matrix-rootfs: add jffs2_extra_args
[matrix.git] / matrix / rootfs.py
1 # Copyright (C) 2007-2009 Movial Creative Technologies Inc.
2 # Authors: Kalle Vahlman <kalle.vahlman@movial.com>
3 #          Timo Savola
4 #          Tuomas Kulve <tuomas.kulve@movial.com>
5 #          Daniel Bainton <daniel.bainton@movial.com>
6
7 import errno
8 import optparse
9 import os
10 import re
11 import shutil
12 import sys
13
14 from config import config
15 from config import rootfs
16 from config import parse as config_parse
17
18 Error = RuntimeError
19
20 image_type_list = (
21         'jffs2',
22         'ubi',
23 )
24 image_type_default = 'jffs2'
25
26 def main():
27         def append_config_dir(option, opt, value, parser):
28                 config.initial_config_dirs.append(value)
29
30         parser = optparse.OptionParser(usage='%prog [<options>] [<root>]')
31
32         parser.add_option(
33                 '-c', '--config-dir', metavar='DIR',
34                 help='add a directory for finding config files (may be set ' \
35                      'multiple times)',
36                 dest='initial_config_dirs', action='callback',
37                 callback=append_config_dir, type='string')
38
39         parser.add_option(
40                 '-n', '--no-clean',
41                 help='do not clean env.faked and stripped directory',
42                 dest='clean', action='store_false', default=True)
43
44         parser.add_option(
45                 '-s', '--no-strip',
46                 help='do not strip the binaries',
47                 dest='no_strip', action='store_true', default=False)
48
49         parser.add_option(
50                 '-i', '--image-only',
51                 help='generate only a filesystem image',
52                 dest='image_only', action='store_true')
53
54         parser.add_option(
55                 '-T', '--image-type',
56                 help='type of generated filesystem image: %s [default: %s]' % \
57                      (', '.join(image_type_list), image_type_default),
58                 dest='image_type', metavar='TYPE',
59                 choices=image_type_list, default=image_type_default)
60
61         parser.add_option(
62                 '-r', '--rootfs-only',
63                 help='generate only a stripped archive',
64                 dest='rootfs_only', action='store_true')
65
66         parser.add_option(
67                 '-d', '--devrootfs-only',
68                 help='generate only an archive with development files',
69                 dest='devrootfs_only', action='store_true')
70
71         opts, args = parser.parse_args()
72         if len(args) > 1:
73                 parser.print_help()
74                 sys.exit(1)
75
76         config_parse('main')
77         config_parse('rootfs')
78
79         if args:
80                 rootfs.path = args[0]
81                 rootfs.target = None
82         elif not rootfs.path and not rootfs.target:
83                 raise Error('Root path must be defined in a config file or on the command-line')
84
85         build = Builder(rootfs.path, rootfs.target)
86         build.include_paths(rootfs.include_paths)
87         build.include_files(rootfs.include_files)
88         build.filter_paths(rootfs.exclude_paths)
89         build.filter_files(rootfs.exclude_files)
90         build.filter_expressions(rootfs.exclude_expressions)
91         build.add_paths(rootfs.created_paths)
92         build.set_devices(rootfs.devices)
93         build.set_change_mode(rootfs.file_modes)
94         build.set_change_owner(rootfs.directory_owners)
95         build.set_flash_erase_size(config.boards[config.board].flash_erase_size)
96         build.set_flash_pad_size(config.boards[config.board].flash_pad_size)
97
98         build.set_flash_compression(config.boards[config.board].flash_compression)
99         build.set_cleanmarkers(config.boards[config.board].cleanmarkers)
100         build.set_jffs2_extra_args(config.boards[config.board].jffs2_extra_args)
101         build.set_ubifs_leb_size(config.boards[config.board].ubifs_leb_size)
102         build.set_ubifs_max_leb_count(config.boards[config.board].ubifs_max_leb_count)
103         build.set_ubifs_min_io_size(config.boards[config.board].ubifs_min_io_size)
104         build.set_ubinize_config_vol_size(config.boards[config.board].ubinize_config_vol_size)
105
106         build.set_no_strip(opts.no_strip)
107
108         if opts.rootfs_only:
109                 build.generate(opts.clean, build_target="rootfs")
110         elif opts.image_only:
111                 build.generate(opts.clean, build_target=opts.image_type)
112         elif opts.devrootfs_only:
113                 build.generate(opts.clean, build_target="devrootfs")
114         else:
115                 build.generate(opts.clean, build_target="all")
116
117 class Builder(object):
118         builddir = "/tmp"
119         path = None
120         target = None
121
122         # Tools
123         strip = "strip"
124         mknod = "mknod"
125         chmod = "chmod"
126         chown = "chown"
127         mkfs_jffs2 = "mkfs.jffs2"
128         mkfs_ubifs = "mkfs.ubifs"
129         ubinize = "ubinize"
130         tar = "tar"
131         gzip = "gzip"
132         flash_erase_size = 0x4000
133         flash_pad_size = 0x3e00000
134         # Compression to use
135         flash_compression = "lzo"
136         # Logical erase block size
137         ubifs_leb_size = 130944
138         # Maximum logical erase block count
139         ubifs_max_leb_count = 120
140         # Minimum I/O unit size
141         ubifs_min_io_size = 1
142         # Size of the physical eraseblock of the flash
143         ubinize_peb_size = flash_erase_size
144
145         # cleanmarkers: yes/no/unknown
146         cleanmarkers = "unknown"
147
148         # extra jffs2 args
149         jffs2_extra_args = ""
150
151         # Configation values for ubinize
152         ubinize_config_mode = "ubi"
153         ubinize_config_image = None
154         ubinize_config_vol_id = 0
155         ubinize_config_vol_size = 13631488
156         ubinize_config_vol_type = "dynamic"
157         ubinize_config_vol_name = "rootfs"
158         ubinize_config_vol_aligment = 1
159         ubinize_config_vol_flags = "autoresize"
160
161         # Do we strip the binaries? - False strips, True doesn't
162         no_strip = False
163
164         file_list = []
165         remove_list = []
166
167         created_paths = []
168         devices = {}
169         chown_owner = {}
170
171         def __init__(self, rootpath, targetname = None):
172                 while rootpath.endswith(os.sep):
173                         rootpath = rootpath[:-1]
174
175                 self.path = rootpath
176                 self.target = targetname or os.path.basename(rootpath)
177
178         def include_paths(self, paths):
179                 old_dir = os.getcwd()
180                 os.chdir(self.path)
181
182                 # Traverse include_paths hierarchies and grab all files there
183                 for path in paths:
184                         for root, dirs, files in os.walk(self.path + path):
185                                 for f in files:
186                                         self.file_list.append("%s/%s" % (root, f))
187                                 if len(files) == 0:
188                                         self.file_list.append(root)
189
190                 os.chdir(old_dir)
191
192         def include_files(self, files):
193                 old_dir = os.getcwd()
194                 os.chdir(self.path)
195
196                 # Append included individual files
197                 for f in files:
198                         self.file_list.append(f)
199
200                 os.chdir(old_dir)
201
202         def filter_paths(self, paths):
203                 # Filter out files in excluded paths
204                 for f in self.file_list:
205                         for d in paths:
206                                 if f == self.path + d or f.startswith(self.path + d + '/'):
207                                         self.remove_list.append(f)
208
209         def filter_files(self, files):
210                 # Filter out excluded files
211                 for f in files:
212                         self.remove_list.append(self.path + f)
213
214         def filter_expressions(self, expressions):
215                 # Filter out files matching excluded patterns
216                 compiled_patterns = []
217                 for pattern_str in expressions:
218                         compiled_patterns.append(re.compile(pattern_str))
219
220                 for f in self.file_list:
221                         for pattern in compiled_patterns:
222                                 if pattern.search(f):
223                                         self.remove_list.append(f)
224
225         def add_paths(self, paths):
226                 for path in paths:
227                         self.created_paths.append(path)
228
229         def set_devices(self, devices):
230                 if devices is not None:
231                         self.devices = devices
232                         
233         def set_change_mode(self, change_mode):
234                 if change_mode is not None:
235                         self.change_mode = change_mode
236
237         def set_change_owner(self, change_owner):
238                 if change_owner is not None:
239                         self.change_owner = change_owner
240
241         def set_flash_erase_size(self, size):
242                 if size is not None:
243                         self.flash_erase_size = size
244                         self.ubinize_peb_size = size
245
246         def set_flash_pad_size(self, size):
247                 if size is not None:
248                         self.flash_pad_size = size
249
250         def set_flash_compression(self, compr):
251                 if compr is not None:
252                         self.flash_compression = compr
253
254         def set_ubifs_leb_size(self, size):
255                 if size is not None:
256                         self.ubifs_leb_size = size
257
258         def set_ubifs_max_leb_count(self, count):
259                 if count is not None:
260                         self.ubifs_max_leb_count = count
261
262         def set_ubifs_min_io_size(self, size):
263                 if size is not None:
264                         self.ubifs_min_io_size = size
265
266         def set_ubinize_config_vol_size(self, size):
267                 if size is not None:
268                         self.ubinize_config_vol_size = size
269
270         def set_cleanmarkers(self, setting):
271                 if setting is "no":
272                         self.cleanmarkers = "-n"
273                 elif setting is "yes":
274                         self.cleanmarkers = ""
275                 elif setting is "unknown":
276                         self.cleanmarkers = "-n"
277                         print "no cleanmarkers setting in the board file, assuming no cleanmarkers"
278                 else:
279                         raise Error("invalid cleanmarkers setting")
280
281         def set_jffs2_extra_args(self, setting):
282                 self.jffs2_extra_args = setting
283
284         def set_no_strip(self, no_strip):
285                 self.no_strip = no_strip
286
287         def generate(self, clean, build_target="all"):
288                 copy_list = self.file_list
289
290                 for f in self.remove_list:
291                         try:
292                                 copy_list.remove(f)
293                         except:
294                                 pass
295
296                 n_files = len(copy_list)
297                 current_percent = -1
298                 current_file = 0
299
300                 old_dir = os.getcwd()
301                 os.chdir(self.path)
302
303                 basepath = os.path.join(self.builddir, self.target)
304
305                 if os.path.exists(basepath):
306                         print basepath, "exists, please remove it and try again"
307                         return
308
309                 # Copy the wanted files, strip binaries
310                 for f in copy_list:
311                         last_percent = current_percent
312                         current_percent = current_file * 100 / n_files
313                         if current_percent != last_percent:
314                                 print "\rCopying files: %3d%% " % current_percent,
315                                 sys.stdout.flush()
316
317                         f = f.replace(self.path, "", 1)
318                         if f[0] == '/':
319                                 f = f[1:]
320                         path = os.path.join(basepath, os.path.dirname(f))
321                         try:
322                                 os.makedirs(path)
323                         except OSError, e:
324                                 if e.errno == errno.EEXIST:
325                                         pass
326
327                         if os.path.islink(f):
328                                 link_dest = os.readlink(f)
329                                 try:
330                                         os.symlink(link_dest, os.path.join(basepath, f))
331                                 except OSError, e:
332                                         if e.errno == errno.EEXIST:
333                                                 pass
334                         elif os.path.isdir(f):
335                                 try:
336                                         os.makedirs(os.path.join(basepath, f))
337                                 except OSError, e:
338                                         if e.errno == errno.EEXIST:
339                                                 pass
340                         else:
341                                 try:
342                                         shutil.copy(f, path)
343                                         if not f.endswith(".ko") and not self.no_strip:
344                                                 os.system(self.strip + " '" + os.path.join(path, os.path.basename(f)) + "' &> /dev/null")
345                                 except IOError, e:
346                                         # We allow nonexistent files since only place where we'll get those
347                                         # is from the include files directive
348                                         if e.errno == errno.EEXIST:
349                                                 pass
350
351                         current_file += 1
352
353                 print "\rCopying files: 100%"
354
355                 # Create extra directory structure
356                 print "Creating extra directories..."
357                 for d in self.created_paths:
358                         if d[0] == '/':
359                                 d = d[1:]
360                         path = os.path.join(basepath, d)
361                         try:
362                                 os.makedirs(path)
363                         except OSError, e:
364                                 if e.errno == errno.EEXIST:
365                                         pass
366
367                 # Remove always the fakeroot environment.
368                 if os.path.exists("/tmp/env.faked"):
369                         os.remove("/tmp/env.faked")
370
371                 fakedbs = []
372                 # TODO: find fakedbs
373
374                 if fakedbs:
375                         os.system('cat >/tmp/env.faked ' + ' '.join(fakedbs))
376                 else:
377                         os.system('touch /tmp/env.faked')
378
379                 # Remove temporary accounts
380
381                 def rewrite_accounts(path, filter):
382                         file = open(path)
383                         lines = [line for line in file if filter(line)]
384                         file.close()
385
386                         file = open(path, 'w')
387                         for line in lines:
388                                 print >>file, line,
389                         file.close()
390
391                 def filter_passwd(line):
392                         return int(line.split(':')[2]) != rootfs.remove_uid
393
394                 def filter_group(line):
395                         return int(line.split(':')[2]) != rootfs.remove_gid
396
397                 passwd_path = os.path.join(basepath, 'etc/passwd')
398                 if os.path.exists(passwd_path) and rootfs.remove_uid is not None:
399                         rewrite_accounts(passwd_path, filter_passwd)
400
401                 group_path = os.path.join(basepath, 'etc/group')
402                 if os.path.exists(group_path) and rootfs.remove_gid is not None:
403                         rewrite_accounts(group_path, filter_group)
404
405                 # Create device nodes
406                 print "Creating device nodes..."
407                 for d in self.devices.keys():
408                         mode = "664"
409                         if len(self.devices[d]) == 3:
410                                 device_type,major,minor = self.devices[d]
411                         else:
412                                 device_type,major,minor,mode = self.devices[d]
413                         os.system("%s -m %s '%s' %s %i %i" % (self.mknod, mode, os.path.join(basepath, 'dev', d), device_type, major, minor))
414
415                 # Fix permissions
416                 print "Adjusting ownerships and permissions..."
417                 # Make the whole fs root-owned
418                 os.system("%s root.root -R %s" % (self.chown, basepath))
419
420                 # For sshd:
421                 os.system("%s go-r %s/etc/ssh*key*" % (self.chmod, basepath))
422
423                 # For /var/empty (sshd requires this):
424                 os.system("%s go-rw /%s/var/empty" % (self.chmod, basepath))
425
426                 # /dev/null needs to be writable by all:
427                 os.system("%s a+rw /%s/dev/null" % (self.chmod, basepath))
428
429                 for d in self.change_mode.keys():
430                         dir = d
431                         if dir[0] =='/':
432                                 dir = dir[1:]
433                         if os.path.exists(os.path.join(basepath, dir)):
434                                 mode = self.change_mode[d]
435                                 os.system("%s %s %s/%s" % (self.chmod, mode, basepath, dir))
436
437                 # Set non-root owners
438                 print "Setting non-root ownerships..."
439                 for d in self.change_owner.keys():
440                         dir = d
441                         if dir[0] == '/':
442                                 dir = dir[1:]
443                         if os.path.exists(os.path.join(basepath, dir)):
444                                 user,group = self.change_owner[d]
445                                 os.system("%s %s.%s -R '%s/%s' " % (self.chown, user, group, basepath, dir))
446
447                 # Build rootfs
448                 if build_target == "all" or build_target == "rootfs":
449                         print "Creating a rootfs..."
450                         os.system("%s -c --one-file-system -C '%s' -z -f '%s.tar.gz' ." % (self.tar, basepath, basepath))
451
452                 # Build jffs2
453                 if build_target == "all" or build_target == "jffs2":
454                         print "Creating a root image as JFFS2..."
455                         os.system("%s -p%d %s -e%d %s -r '%s' -o '%s.jffs2'" % (self.mkfs_jffs2, self.flash_pad_size, self.cleanmarkers, self.flash_erase_size, self.jffs2_extra_args, basepath, basepath))
456
457                 # Build ubifs
458                 if build_target == "ubi":
459                         print "Creating a root image as UBI..."
460                         ubinize_config = ""
461                         ubinize_config += "[ubifs]\n"
462                         ubinize_config += "mode=%s\n" % self.ubinize_config_mode
463                         ubinize_config += "image=%s.ubifs\n" % (basepath)
464                         ubinize_config += "vol_id=%d\n" % self.ubinize_config_vol_id
465                         ubinize_config += "vol_size=%d\n" % self.ubinize_config_vol_size
466                         ubinize_config += "vol_type=%s\n" % self.ubinize_config_vol_type
467                         ubinize_config += "vol_name=%s\n" % self.ubinize_config_vol_name
468                         ubinize_config += "vol_alignment=%d\n" % self.ubinize_config_vol_aligment
469                         ubinize_config += "vol_flags=%s\n" % self.ubinize_config_vol_flags
470                         f=open('/tmp/ubinize.cfg', 'w')
471                         f.write(ubinize_config)
472                         f.close()
473
474                         os.system("%s --compr=%s -r '%s' -o '%s.ubifs' -m %d -e %d -c %d" % (self.mkfs_ubifs, self.flash_compression, basepath, basepath, self.ubifs_min_io_size, self.ubifs_leb_size, self.ubifs_max_leb_count))
475                         os.system("%s -m %d -p %d -o '%s.ubi' '%s/ubinize.cfg'" % (self.ubinize, self.ubifs_min_io_size, self.ubinize_peb_size, basepath, self.builddir))
476
477                 # Build development rootfs
478                 if build_target == "all" or build_target == "devrootfs":
479                         print "Creating a rootstrap..."
480                         os.system("%s -c --one-file-system -C '%s' -f '%s-rootstrap.tar' ." % (self.tar, self.path, basepath))
481                         # override passwd/group with the modified versions
482                         os.system("%s -r -C '%s' -f '%s-rootstrap.tar' ./etc/passwd ./etc/group" % (self.tar, basepath, basepath))
483                         os.system("%s -f '%s-rootstrap.tar'" % (self.gzip, basepath))
484
485                 os.chdir(old_dir)
486
487                 print "Build finished:"
488                 for targets, suffix in ((('rootfs', 'all'), '.tar.gz'),
489                                         (('jffs2', 'all'), '.jffs2'),
490                                         (('ubi',), '.ubi'),
491                                         (('devrootfs', 'all'), '-rootstrap.tar.gz')):
492                         if build_target in targets:
493                                 fullname = basepath + suffix
494                                 s = os.stat(fullname)
495                                 size = s.st_size / 1024.0 / 1024.0
496                                 print '  %-40s %.2f MiB' % (fullname, size)
497
498                 if clean:
499                         print "Cleaning up"
500                         os.remove("/tmp/env.faked")
501                         os.system("rm -rf '%s'" % basepath)