a2090d8edd659874e4fccfd5aabef981028eb341
[matrix.git] / matrix / rootfs.py
1 # Copyright (C) 2007-2009 Movial Creative Technologies Inc.
2 # Authors: Kalle Vahlman <kalle.vahlman@movial.com>
3 #          Timo Savola
4 #          Tuomas Kulve <tuomas.kulve@movial.com>
5 #          Daniel Bainton <daniel.bainton@movial.com>
6
7 import errno
8 import optparse
9 import os
10 import re
11 import shutil
12 import sys
13
14 from config import config
15 from config import rootfs
16 from config import parse as config_parse
17
18 Error = RuntimeError
19
20 image_type_list = (
21         'jffs2',
22         'ubi',
23 )
24 image_type_default = 'jffs2'
25
26 def main():
27         def append_config_dir(option, opt, value, parser):
28                 config.initial_config_dirs.append(value)
29
30         parser = optparse.OptionParser(usage='%prog [<options>] [<root>]')
31
32         parser.add_option(
33                 '-c', '--config-dir', metavar='DIR',
34                 help='add a directory for finding config files (may be set ' \
35                      'multiple times)',
36                 dest='initial_config_dirs', action='callback',
37                 callback=append_config_dir, type='string')
38
39         parser.add_option(
40                 '-n', '--no-clean',
41                 help='do not clean env.faked and stripped directory',
42                 dest='clean', action='store_false', default=True)
43
44         parser.add_option(
45                 '-s', '--no-strip',
46                 help='do not strip the binaries',
47                 dest='no_strip', action='store_true', default=False)
48
49         parser.add_option(
50                 '-i', '--image-only',
51                 help='generate only a filesystem image',
52                 dest='image_only', action='store_true')
53
54         parser.add_option(
55                 '-T', '--image-type',
56                 help='type of generated filesystem image: %s [default: %s]' % \
57                      (', '.join(image_type_list), image_type_default),
58                 dest='image_type', metavar='TYPE',
59                 choices=image_type_list, default=image_type_default)
60
61         parser.add_option(
62                 '-r', '--rootfs-only',
63                 help='generate only a stripped archive',
64                 dest='rootfs_only', action='store_true')
65
66         parser.add_option(
67                 '-d', '--devrootfs-only',
68                 help='generate only an archive with development files',
69                 dest='devrootfs_only', action='store_true')
70
71         opts, args = parser.parse_args()
72         if len(args) > 1:
73                 parser.print_help()
74                 sys.exit(1)
75
76         config_parse('main')
77         config_parse('rootfs')
78
79         if args:
80                 rootfs.path = args[0]
81                 rootfs.target = None
82         elif not rootfs.path and not rootfs.target:
83                 raise Error('Root path must be defined in a config file or on the command-line')
84
85         build = Builder(rootfs.path, rootfs.target)
86         build.include_paths(rootfs.include_paths)
87         build.include_files(rootfs.include_files)
88         build.filter_paths(rootfs.exclude_paths)
89         build.filter_files(rootfs.exclude_files)
90         build.filter_expressions(rootfs.exclude_expressions)
91         build.add_paths(rootfs.created_paths)
92         build.set_devices(rootfs.devices)
93         build.set_change_mode(rootfs.file_modes)
94         build.set_change_owner(rootfs.directory_owners)
95         build.set_flash_erase_size(config.boards[config.board].flash_erase_size)
96         build.set_flash_pad_size(config.boards[config.board].flash_pad_size)
97
98         build.set_flash_compression(config.boards[config.board].flash_compression)
99         build.set_ubifs_leb_size(config.boards[config.board].ubifs_leb_size)
100         build.set_ubifs_max_leb_count(config.boards[config.board].ubifs_max_leb_count)
101         build.set_ubifs_min_io_size(config.boards[config.board].ubifs_min_io_size)
102         build.set_ubinize_config_vol_size(config.boards[config.board].ubinize_config_vol_size)
103
104         build.set_no_strip(opts.no_strip)
105
106         if opts.rootfs_only:
107                 build.generate(opts.clean, build_target="rootfs")
108         elif opts.image_only:
109                 build.generate(opts.clean, build_target=opts.image_type)
110         elif opts.devrootfs_only:
111                 build.generate(opts.clean, build_target="devrootfs")
112         else:
113                 build.generate(opts.clean, build_target="all")
114
115 class Builder(object):
116         builddir = "/tmp"
117         path = None
118         target = None
119
120         # Tools
121         strip = "strip"
122         mknod = "mknod"
123         chmod = "chmod"
124         chown = "chown"
125         mkfs_jffs2 = "mkfs.jffs2"
126         mkfs_ubifs = "mkfs.ubifs"
127         ubinize = "ubinize"
128         tar = "tar"
129         gzip = "gzip"
130         flash_erase_size = 0x4000
131         flash_pad_size = 0x3e00000
132         # Compression to use
133         flash_compression = "lzo"
134         # Logical erase block size
135         ubifs_leb_size = 130944
136         # Maximum logical erase block count
137         ubifs_max_leb_count = 120
138         # Minimum I/O unit size
139         ubifs_min_io_size = 1
140         # Size of the physical eraseblock of the flash
141         ubinize_peb_size = flash_erase_size
142
143         # Configation values for ubinize
144         ubinize_config_mode = "ubi"
145         ubinize_config_image = None
146         ubinize_config_vol_id = 0
147         ubinize_config_vol_size = 13631488
148         ubinize_config_vol_type = "dynamic"
149         ubinize_config_vol_name = "rootfs"
150         ubinize_config_vol_aligment = 1
151         ubinize_config_vol_flags = "autoresize"
152
153         # Do we strip the binaries? - False strips, True doesn't
154         no_strip = False
155
156         file_list = []
157         remove_list = []
158
159         created_paths = []
160         devices = {}
161         chown_owner = {}
162
163         def __init__(self, rootpath, targetname = None):
164                 while rootpath.endswith(os.sep):
165                         rootpath = rootpath[:-1]
166
167                 self.path = rootpath
168                 self.target = targetname or os.path.basename(rootpath)
169
170         def include_paths(self, paths):
171                 old_dir = os.getcwd()
172                 os.chdir(self.path)
173
174                 # Traverse include_paths hierarchies and grab all files there
175                 for path in paths:
176                         for root, dirs, files in os.walk(self.path + path):
177                                 for f in files:
178                                         self.file_list.append("%s/%s" % (root, f))
179                                 if len(files) == 0:
180                                         self.file_list.append(root)
181
182                 os.chdir(old_dir)
183
184         def include_files(self, files):
185                 old_dir = os.getcwd()
186                 os.chdir(self.path)
187
188                 # Append included individual files
189                 for f in files:
190                         self.file_list.append(f)
191
192                 os.chdir(old_dir)
193
194         def filter_paths(self, paths):
195                 # Filter out files in excluded paths
196                 for f in self.file_list:
197                         for d in paths:
198                                 if f == self.path + d or f.startswith(self.path + d + '/'):
199                                         self.remove_list.append(f)
200
201         def filter_files(self, files):
202                 # Filter out excluded files
203                 for f in files:
204                         self.remove_list.append(self.path + f)
205
206         def filter_expressions(self, expressions):
207                 # Filter out files matching excluded patterns
208                 compiled_patterns = []
209                 for pattern_str in expressions:
210                         compiled_patterns.append(re.compile(pattern_str))
211
212                 for f in self.file_list:
213                         for pattern in compiled_patterns:
214                                 if pattern.search(f):
215                                         self.remove_list.append(f)
216
217         def add_paths(self, paths):
218                 for path in paths:
219                         self.created_paths.append(path)
220
221         def set_devices(self, devices):
222                 if devices is not None:
223                         self.devices = devices
224                         
225         def set_change_mode(self, change_mode):
226                 if change_mode is not None:
227                         self.change_mode = change_mode
228
229         def set_change_owner(self, change_owner):
230                 if change_owner is not None:
231                         self.change_owner = change_owner
232
233         def set_flash_erase_size(self, size):
234                 if size is not None:
235                         self.flash_erase_size = size
236                         self.ubinize_peb_size = size
237
238         def set_flash_pad_size(self, size):
239                 if size is not None:
240                         self.flash_pad_size = size
241
242         def set_flash_compression(self, compr):
243                 if compr is not None:
244                         self.flash_compression = compr
245
246         def set_ubifs_leb_size(self, size):
247                 if size is not None:
248                         self.ubifs_leb_size = size
249
250         def set_ubifs_max_leb_count(self, count):
251                 if count is not None:
252                         self.ubifs_max_leb_count = count
253
254         def set_ubifs_min_io_size(self, size):
255                 if size is not None:
256                         self.ubifs_min_io_size = size
257
258         def set_ubinize_config_vol_size(self, size):
259                 if size is not None:
260                         self.ubinize_config_vol_size = size
261
262         def set_no_strip(self, no_strip):
263                 self.no_strip = no_strip
264
265         def generate(self, clean, build_target="all"):
266                 copy_list = self.file_list
267
268                 for f in self.remove_list:
269                         try:
270                                 copy_list.remove(f)
271                         except:
272                                 pass
273
274                 n_files = len(copy_list)
275                 current_percent = -1
276                 current_file = 0
277
278                 old_dir = os.getcwd()
279                 os.chdir(self.path)
280
281                 basepath = os.path.join(self.builddir, self.target)
282
283                 if os.path.exists(basepath):
284                         print basepath, "exists, please remove it and try again"
285                         return
286
287                 # Copy the wanted files, strip binaries
288                 for f in copy_list:
289                         last_percent = current_percent
290                         current_percent = current_file * 100 / n_files
291                         if current_percent != last_percent:
292                                 print "\rCopying files: %3d%% " % current_percent,
293                                 sys.stdout.flush()
294
295                         f = f.replace(self.path, "", 1)
296                         if f[0] == '/':
297                                 f = f[1:]
298                         path = os.path.join(basepath, os.path.dirname(f))
299                         try:
300                                 os.makedirs(path)
301                         except OSError, e:
302                                 if e.errno == errno.EEXIST:
303                                         pass
304
305                         if os.path.islink(f):
306                                 link_dest = os.readlink(f)
307                                 try:
308                                         os.symlink(link_dest, os.path.join(basepath, f))
309                                 except OSError, e:
310                                         if e.errno == errno.EEXIST:
311                                                 pass
312                         elif os.path.isdir(f):
313                                 try:
314                                         os.makedirs(os.path.join(basepath, f))
315                                 except OSError, e:
316                                         if e.errno == errno.EEXIST:
317                                                 pass
318                         else:
319                                 try:
320                                         shutil.copy(f, path)
321                                         if not f.endswith(".ko") and not self.no_strip:
322                                                 os.system(self.strip + " '" + os.path.join(path, os.path.basename(f)) + "' &> /dev/null")
323                                 except IOError, e:
324                                         # We allow nonexistent files since only place where we'll get those
325                                         # is from the include files directive
326                                         if e.errno == errno.EEXIST:
327                                                 pass
328
329                         current_file += 1
330
331                 print "\rCopying files: 100%"
332
333                 # Create extra directory structure
334                 print "Creating extra directories..."
335                 for d in self.created_paths:
336                         if d[0] == '/':
337                                 d = d[1:]
338                         path = os.path.join(basepath, d)
339                         try:
340                                 os.makedirs(path)
341                         except OSError, e:
342                                 if e.errno == errno.EEXIST:
343                                         pass
344
345                 # Remove always the fakeroot environment.
346                 if os.path.exists("/tmp/env.faked"):
347                         os.remove("/tmp/env.faked")
348
349                 fakedbs = []
350                 # TODO: find fakedbs
351
352                 if fakedbs:
353                         os.system('cat >/tmp/env.faked ' + ' '.join(fakedbs))
354                 else:
355                         os.system('touch /tmp/env.faked')
356
357                 # Remove temporary accounts
358
359                 def rewrite_accounts(path, filter):
360                         file = open(path)
361                         lines = [line for line in file if filter(line)]
362                         file.close()
363
364                         file = open(path, 'w')
365                         for line in lines:
366                                 print >>file, line,
367                         file.close()
368
369                 def filter_passwd(line):
370                         return int(line.split(':')[2]) != rootfs.remove_uid
371
372                 def filter_group(line):
373                         return int(line.split(':')[2]) != rootfs.remove_gid
374
375                 passwd_path = os.path.join(basepath, 'etc/passwd')
376                 if os.path.exists(passwd_path) and rootfs.remove_uid is not None:
377                         rewrite_accounts(passwd_path, filter_passwd)
378
379                 group_path = os.path.join(basepath, 'etc/group')
380                 if os.path.exists(group_path) and rootfs.remove_gid is not None:
381                         rewrite_accounts(group_path, filter_group)
382
383                 # Create device nodes
384                 print "Creating device nodes..."
385                 for d in self.devices.keys():
386                         mode = "664"
387                         if len(self.devices[d]) == 3:
388                                 device_type,major,minor = self.devices[d]
389                         else:
390                                 device_type,major,minor,mode = self.devices[d]
391                         os.system("%s -m %s '%s' %s %i %i" % (self.mknod, mode, os.path.join(basepath, 'dev', d), device_type, major, minor))
392
393                 # Fix permissions
394                 print "Adjusting ownerships and permissions..."
395                 # Make the whole fs root-owned
396                 os.system("%s root.root -R %s" % (self.chown, basepath))
397
398                 # For sshd:
399                 os.system("%s go-r %s/etc/ssh*key*" % (self.chmod, basepath))
400
401                 # For /var/empty (sshd requires this):
402                 os.system("%s go-rw /%s/var/empty" % (self.chmod, basepath))
403
404                 # /dev/null needs to be writable by all:
405                 os.system("%s a+rw /%s/dev/null" % (self.chmod, basepath))
406
407                 for d in self.change_mode.keys():
408                         dir = d
409                         if dir[0] =='/':
410                                 dir = dir[1:]
411                         if os.path.exists(os.path.join(basepath, dir)):
412                                 mode = self.change_mode[d]
413                                 os.system("%s %s %s/%s" % (self.chmod, mode, basepath, dir))
414
415                 # Set non-root owners
416                 print "Setting non-root ownerships..."
417                 for d in self.change_owner.keys():
418                         dir = d
419                         if dir[0] == '/':
420                                 dir = dir[1:]
421                         if os.path.exists(os.path.join(basepath, dir)):
422                                 user,group = self.change_owner[d]
423                                 os.system("%s %s.%s -R '%s/%s' " % (self.chown, user, group, basepath, dir))
424
425                 # Build rootfs
426                 if build_target == "all" or build_target == "rootfs":
427                         print "Creating a rootfs..."
428                         os.system("%s -c --one-file-system -C '%s' -z -f '%s.tar.gz' ." % (self.tar, basepath, basepath))
429
430                 # Build jffs2
431                 if build_target == "all" or build_target == "jffs2":
432                         print "Creating a root image as JFFS2..."
433                         os.system("%s -p%d -n -e%d -r '%s' -o '%s.jffs2'" % (self.mkfs_jffs2, self.flash_pad_size, self.flash_erase_size, basepath, basepath))
434
435                 # Build ubifs
436                 if build_target == "ubi":
437                         print "Creating a root image as UBI..."
438                         ubinize_config = ""
439                         ubinize_config += "[ubifs]\n"
440                         ubinize_config += "mode=%s\n" % self.ubinize_config_mode
441                         ubinize_config += "image=%s.ubifs\n" % (basepath)
442                         ubinize_config += "vol_id=%d\n" % self.ubinize_config_vol_id
443                         ubinize_config += "vol_size=%d\n" % self.ubinize_config_vol_size
444                         ubinize_config += "vol_type=%s\n" % self.ubinize_config_vol_type
445                         ubinize_config += "vol_name=%s\n" % self.ubinize_config_vol_name
446                         ubinize_config += "vol_alignment=%d\n" % self.ubinize_config_vol_aligment
447                         ubinize_config += "vol_flags=%s\n" % self.ubinize_config_vol_flags
448                         f=open('/tmp/ubinize.cfg', 'w')
449                         f.write(ubinize_config)
450                         f.close()
451
452                         os.system("%s --compr=%s -r '%s' -o '%s.ubifs' -m %d -e %d -c %d" % (self.mkfs_ubifs, self.flash_compression, basepath, basepath, self.ubifs_min_io_size, self.ubifs_leb_size, self.ubifs_max_leb_count))
453                         os.system("%s -m %d -p %d -o '%s.ubi' '%s/ubinize.cfg'" % (self.ubinize, self.ubifs_min_io_size, self.ubinize_peb_size, basepath, self.builddir))
454
455                 # Build development rootfs
456                 if build_target == "all" or build_target == "devrootfs":
457                         print "Creating a rootstrap..."
458                         os.system("%s -c --one-file-system -C '%s' -f '%s-rootstrap.tar' ." % (self.tar, self.path, basepath))
459                         # override passwd/group with the modified versions
460                         os.system("%s -r -C '%s' -f '%s-rootstrap.tar' ./etc/passwd ./etc/group" % (self.tar, basepath, basepath))
461                         os.system("%s -f '%s-rootstrap.tar'" % (self.gzip, basepath))
462
463                 os.chdir(old_dir)
464
465                 print "Build finished:"
466                 for targets, suffix in ((('rootfs', 'all'), '.tar.gz'),
467                                         (('jffs2', 'all'), '.jffs2'),
468                                         (('ubi',), '.ubi'),
469                                         (('devrootfs', 'all'), '-rootstrap.tar.gz')):
470                         if build_target in targets:
471                                 fullname = basepath + suffix
472                                 s = os.stat(fullname)
473                                 size = s.st_size / 1024.0 / 1024.0
474                                 print '  %-40s %.2f MiB' % (fullname, size)
475
476                 if clean:
477                         print "Cleaning up"
478                         os.remove("/tmp/env.faked")
479                         os.system("rm -rf '%s'" % basepath)