5fd0bda501089f2e496ed7025e2768b8bafffd40
[matrix.git] / matrix / rootfs.py
1 # Copyright (C) 2007-2009 Movial Creative Technologies Inc.
2 # Authors: Kalle Vahlman <kalle.vahlman@movial.com>
3 #          Timo Savola
4 #          Tuomas Kulve <tuomas.kulve@movial.com>
5 #          Daniel Bainton <daniel.bainton@movial.com>
6
7 import errno
8 import optparse
9 import os
10 import re
11 import shutil
12 import sys
13
14 from config import config
15 from config import rootfs
16 from config import parse as config_parse
17
18 Error = RuntimeError
19
20 image_type_list = (
21         'jffs2',
22         'ubi',
23 )
24 image_type_default = 'jffs2'
25
26 def main():
27         def append_config_dir(option, opt, value, parser):
28                 config.initial_config_dirs.append(value)
29
30         parser = optparse.OptionParser(usage='%prog [<options>] [<root>]')
31
32         parser.add_option(
33                 '-c', '--config-dir', metavar='DIR',
34                 help='add a directory for finding config files (may be set ' \
35                      'multiple times)',
36                 dest='initial_config_dirs', action='callback',
37                 callback=append_config_dir, type='string')
38
39         parser.add_option(
40                 '-n', '--no-clean',
41                 help='do not clean env.faked and stripped directory',
42                 dest='clean', action='store_false', default=True)
43
44         parser.add_option(
45                 '-s', '--no-strip',
46                 help='do not strip the binaries',
47                 dest='no_strip', action='store_true', default=False)
48
49         parser.add_option(
50                 '-i', '--image-only',
51                 help='generate only a filesystem image',
52                 dest='image_only', action='store_true')
53
54         parser.add_option(
55                 '-T', '--image-type',
56                 help='type of generated filesystem image: %s [default: %s]' % \
57                      (', '.join(image_type_list), image_type_default),
58                 dest='image_type', metavar='TYPE',
59                 choices=image_type_list, default=image_type_default)
60
61         parser.add_option(
62                 '-r', '--rootfs-only',
63                 help='generate only a stripped archive',
64                 dest='rootfs_only', action='store_true')
65
66         parser.add_option(
67                 '-d', '--devrootfs-only',
68                 help='generate only an archive with development files',
69                 dest='devrootfs_only', action='store_true')
70
71         opts, args = parser.parse_args()
72         if len(args) > 1:
73                 parser.print_help()
74                 sys.exit(1)
75
76         config_parse('main')
77         config_parse('rootfs')
78
79         if args:
80                 rootfs.path = args[0]
81                 rootfs.target = None
82         elif not rootfs.path and not rootfs.target:
83                 raise Error('Root path must be defined in a config file or on the command-line')
84
85         build = Builder(rootfs.path, rootfs.target)
86         build.include_paths(rootfs.include_paths)
87         build.include_files(rootfs.include_files)
88         build.filter_paths(rootfs.exclude_paths)
89         build.filter_files(rootfs.exclude_files)
90         build.filter_expressions(rootfs.exclude_expressions)
91         build.add_paths(rootfs.created_paths)
92         build.set_devices(rootfs.devices)
93         build.set_change_mode(rootfs.file_modes)
94         build.set_change_owner(rootfs.directory_owners)
95         build.set_flash_erase_size(config.boards[config.board].flash_erase_size)
96         build.set_flash_pad_size(config.boards[config.board].flash_pad_size)
97
98         build.set_flash_compression(config.boards[config.board].flash_compression)
99         build.set_cleanmarkers(config.boards[config.board].cleanmarkers)
100         build.set_ubifs_leb_size(config.boards[config.board].ubifs_leb_size)
101         build.set_ubifs_max_leb_count(config.boards[config.board].ubifs_max_leb_count)
102         build.set_ubifs_min_io_size(config.boards[config.board].ubifs_min_io_size)
103         build.set_ubinize_config_vol_size(config.boards[config.board].ubinize_config_vol_size)
104
105         build.set_no_strip(opts.no_strip)
106
107         if opts.rootfs_only:
108                 build.generate(opts.clean, build_target="rootfs")
109         elif opts.image_only:
110                 build.generate(opts.clean, build_target=opts.image_type)
111         elif opts.devrootfs_only:
112                 build.generate(opts.clean, build_target="devrootfs")
113         else:
114                 build.generate(opts.clean, build_target="all")
115
116 class Builder(object):
117         builddir = "/tmp"
118         path = None
119         target = None
120
121         # Tools
122         strip = "strip"
123         mknod = "mknod"
124         chmod = "chmod"
125         chown = "chown"
126         mkfs_jffs2 = "mkfs.jffs2"
127         mkfs_ubifs = "mkfs.ubifs"
128         ubinize = "ubinize"
129         tar = "tar"
130         gzip = "gzip"
131         flash_erase_size = 0x4000
132         flash_pad_size = 0x3e00000
133         # Compression to use
134         flash_compression = "lzo"
135         # Logical erase block size
136         ubifs_leb_size = 130944
137         # Maximum logical erase block count
138         ubifs_max_leb_count = 120
139         # Minimum I/O unit size
140         ubifs_min_io_size = 1
141         # Size of the physical eraseblock of the flash
142         ubinize_peb_size = flash_erase_size
143
144         # cleanmarkers: yes/no/unknown
145         cleanmarkers = "unknown"
146
147         # Configation values for ubinize
148         ubinize_config_mode = "ubi"
149         ubinize_config_image = None
150         ubinize_config_vol_id = 0
151         ubinize_config_vol_size = 13631488
152         ubinize_config_vol_type = "dynamic"
153         ubinize_config_vol_name = "rootfs"
154         ubinize_config_vol_aligment = 1
155         ubinize_config_vol_flags = "autoresize"
156
157         # Do we strip the binaries? - False strips, True doesn't
158         no_strip = False
159
160         file_list = []
161         remove_list = []
162
163         created_paths = []
164         devices = {}
165         chown_owner = {}
166
167         def __init__(self, rootpath, targetname = None):
168                 while rootpath.endswith(os.sep):
169                         rootpath = rootpath[:-1]
170
171                 self.path = rootpath
172                 self.target = targetname or os.path.basename(rootpath)
173
174         def include_paths(self, paths):
175                 old_dir = os.getcwd()
176                 os.chdir(self.path)
177
178                 # Traverse include_paths hierarchies and grab all files there
179                 for path in paths:
180                         for root, dirs, files in os.walk(self.path + path):
181                                 for f in files:
182                                         self.file_list.append("%s/%s" % (root, f))
183                                 if len(files) == 0:
184                                         self.file_list.append(root)
185
186                 os.chdir(old_dir)
187
188         def include_files(self, files):
189                 old_dir = os.getcwd()
190                 os.chdir(self.path)
191
192                 # Append included individual files
193                 for f in files:
194                         self.file_list.append(f)
195
196                 os.chdir(old_dir)
197
198         def filter_paths(self, paths):
199                 # Filter out files in excluded paths
200                 for f in self.file_list:
201                         for d in paths:
202                                 if f == self.path + d or f.startswith(self.path + d + '/'):
203                                         self.remove_list.append(f)
204
205         def filter_files(self, files):
206                 # Filter out excluded files
207                 for f in files:
208                         self.remove_list.append(self.path + f)
209
210         def filter_expressions(self, expressions):
211                 # Filter out files matching excluded patterns
212                 compiled_patterns = []
213                 for pattern_str in expressions:
214                         compiled_patterns.append(re.compile(pattern_str))
215
216                 for f in self.file_list:
217                         for pattern in compiled_patterns:
218                                 if pattern.search(f):
219                                         self.remove_list.append(f)
220
221         def add_paths(self, paths):
222                 for path in paths:
223                         self.created_paths.append(path)
224
225         def set_devices(self, devices):
226                 if devices is not None:
227                         self.devices = devices
228                         
229         def set_change_mode(self, change_mode):
230                 if change_mode is not None:
231                         self.change_mode = change_mode
232
233         def set_change_owner(self, change_owner):
234                 if change_owner is not None:
235                         self.change_owner = change_owner
236
237         def set_flash_erase_size(self, size):
238                 if size is not None:
239                         self.flash_erase_size = size
240                         self.ubinize_peb_size = size
241
242         def set_flash_pad_size(self, size):
243                 if size is not None:
244                         self.flash_pad_size = size
245
246         def set_flash_compression(self, compr):
247                 if compr is not None:
248                         self.flash_compression = compr
249
250         def set_ubifs_leb_size(self, size):
251                 if size is not None:
252                         self.ubifs_leb_size = size
253
254         def set_ubifs_max_leb_count(self, count):
255                 if count is not None:
256                         self.ubifs_max_leb_count = count
257
258         def set_ubifs_min_io_size(self, size):
259                 if size is not None:
260                         self.ubifs_min_io_size = size
261
262         def set_ubinize_config_vol_size(self, size):
263                 if size is not None:
264                         self.ubinize_config_vol_size = size
265
266         def set_cleanmarkers(self, setting):
267                 if setting is "no":
268                         self.cleanmarkers = "-n"
269                 elif setting is "yes":
270                         self.cleanmarkers = ""
271                 elif setting is "unknown":
272                         self.cleanmarkers = "-n"
273                         print "no cleanmarkers setting in the board file, assuming no cleanmarkers"
274                 else:
275                         raise Error("invalid cleanmarkers setting")
276                         
277
278         def set_no_strip(self, no_strip):
279                 self.no_strip = no_strip
280
281         def generate(self, clean, build_target="all"):
282                 copy_list = self.file_list
283
284                 for f in self.remove_list:
285                         try:
286                                 copy_list.remove(f)
287                         except:
288                                 pass
289
290                 n_files = len(copy_list)
291                 current_percent = -1
292                 current_file = 0
293
294                 old_dir = os.getcwd()
295                 os.chdir(self.path)
296
297                 basepath = os.path.join(self.builddir, self.target)
298
299                 if os.path.exists(basepath):
300                         print basepath, "exists, please remove it and try again"
301                         return
302
303                 # Copy the wanted files, strip binaries
304                 for f in copy_list:
305                         last_percent = current_percent
306                         current_percent = current_file * 100 / n_files
307                         if current_percent != last_percent:
308                                 print "\rCopying files: %3d%% " % current_percent,
309                                 sys.stdout.flush()
310
311                         f = f.replace(self.path, "", 1)
312                         if f[0] == '/':
313                                 f = f[1:]
314                         path = os.path.join(basepath, os.path.dirname(f))
315                         try:
316                                 os.makedirs(path)
317                         except OSError, e:
318                                 if e.errno == errno.EEXIST:
319                                         pass
320
321                         if os.path.islink(f):
322                                 link_dest = os.readlink(f)
323                                 try:
324                                         os.symlink(link_dest, os.path.join(basepath, f))
325                                 except OSError, e:
326                                         if e.errno == errno.EEXIST:
327                                                 pass
328                         elif os.path.isdir(f):
329                                 try:
330                                         os.makedirs(os.path.join(basepath, f))
331                                 except OSError, e:
332                                         if e.errno == errno.EEXIST:
333                                                 pass
334                         else:
335                                 try:
336                                         shutil.copy(f, path)
337                                         if not f.endswith(".ko") and not self.no_strip:
338                                                 os.system(self.strip + " '" + os.path.join(path, os.path.basename(f)) + "' &> /dev/null")
339                                 except IOError, e:
340                                         # We allow nonexistent files since only place where we'll get those
341                                         # is from the include files directive
342                                         if e.errno == errno.EEXIST:
343                                                 pass
344
345                         current_file += 1
346
347                 print "\rCopying files: 100%"
348
349                 # Create extra directory structure
350                 print "Creating extra directories..."
351                 for d in self.created_paths:
352                         if d[0] == '/':
353                                 d = d[1:]
354                         path = os.path.join(basepath, d)
355                         try:
356                                 os.makedirs(path)
357                         except OSError, e:
358                                 if e.errno == errno.EEXIST:
359                                         pass
360
361                 # Remove always the fakeroot environment.
362                 if os.path.exists("/tmp/env.faked"):
363                         os.remove("/tmp/env.faked")
364
365                 fakedbs = []
366                 # TODO: find fakedbs
367
368                 if fakedbs:
369                         os.system('cat >/tmp/env.faked ' + ' '.join(fakedbs))
370                 else:
371                         os.system('touch /tmp/env.faked')
372
373                 # Remove temporary accounts
374
375                 def rewrite_accounts(path, filter):
376                         file = open(path)
377                         lines = [line for line in file if filter(line)]
378                         file.close()
379
380                         file = open(path, 'w')
381                         for line in lines:
382                                 print >>file, line,
383                         file.close()
384
385                 def filter_passwd(line):
386                         return int(line.split(':')[2]) != rootfs.remove_uid
387
388                 def filter_group(line):
389                         return int(line.split(':')[2]) != rootfs.remove_gid
390
391                 passwd_path = os.path.join(basepath, 'etc/passwd')
392                 if os.path.exists(passwd_path) and rootfs.remove_uid is not None:
393                         rewrite_accounts(passwd_path, filter_passwd)
394
395                 group_path = os.path.join(basepath, 'etc/group')
396                 if os.path.exists(group_path) and rootfs.remove_gid is not None:
397                         rewrite_accounts(group_path, filter_group)
398
399                 # Create device nodes
400                 print "Creating device nodes..."
401                 for d in self.devices.keys():
402                         mode = "664"
403                         if len(self.devices[d]) == 3:
404                                 device_type,major,minor = self.devices[d]
405                         else:
406                                 device_type,major,minor,mode = self.devices[d]
407                         os.system("%s -m %s '%s' %s %i %i" % (self.mknod, mode, os.path.join(basepath, 'dev', d), device_type, major, minor))
408
409                 # Fix permissions
410                 print "Adjusting ownerships and permissions..."
411                 # Make the whole fs root-owned
412                 os.system("%s root.root -R %s" % (self.chown, basepath))
413
414                 # For sshd:
415                 os.system("%s go-r %s/etc/ssh*key*" % (self.chmod, basepath))
416
417                 # For /var/empty (sshd requires this):
418                 os.system("%s go-rw /%s/var/empty" % (self.chmod, basepath))
419
420                 # /dev/null needs to be writable by all:
421                 os.system("%s a+rw /%s/dev/null" % (self.chmod, basepath))
422
423                 for d in self.change_mode.keys():
424                         dir = d
425                         if dir[0] =='/':
426                                 dir = dir[1:]
427                         if os.path.exists(os.path.join(basepath, dir)):
428                                 mode = self.change_mode[d]
429                                 os.system("%s %s %s/%s" % (self.chmod, mode, basepath, dir))
430
431                 # Set non-root owners
432                 print "Setting non-root ownerships..."
433                 for d in self.change_owner.keys():
434                         dir = d
435                         if dir[0] == '/':
436                                 dir = dir[1:]
437                         if os.path.exists(os.path.join(basepath, dir)):
438                                 user,group = self.change_owner[d]
439                                 os.system("%s %s.%s -R '%s/%s' " % (self.chown, user, group, basepath, dir))
440
441                 # Build rootfs
442                 if build_target == "all" or build_target == "rootfs":
443                         print "Creating a rootfs..."
444                         os.system("%s -c --one-file-system -C '%s' -z -f '%s.tar.gz' ." % (self.tar, basepath, basepath))
445
446                 # Build jffs2
447                 if build_target == "all" or build_target == "jffs2":
448                         print "Creating a root image as JFFS2..."
449                         os.system("%s -p%d %s -e%d -r '%s' -o '%s.jffs2'" % (self.mkfs_jffs2, self.flash_pad_size, self.cleanmarkers, self.flash_erase_size, basepath, basepath))
450
451                 # Build ubifs
452                 if build_target == "ubi":
453                         print "Creating a root image as UBI..."
454                         ubinize_config = ""
455                         ubinize_config += "[ubifs]\n"
456                         ubinize_config += "mode=%s\n" % self.ubinize_config_mode
457                         ubinize_config += "image=%s.ubifs\n" % (basepath)
458                         ubinize_config += "vol_id=%d\n" % self.ubinize_config_vol_id
459                         ubinize_config += "vol_size=%d\n" % self.ubinize_config_vol_size
460                         ubinize_config += "vol_type=%s\n" % self.ubinize_config_vol_type
461                         ubinize_config += "vol_name=%s\n" % self.ubinize_config_vol_name
462                         ubinize_config += "vol_alignment=%d\n" % self.ubinize_config_vol_aligment
463                         ubinize_config += "vol_flags=%s\n" % self.ubinize_config_vol_flags
464                         f=open('/tmp/ubinize.cfg', 'w')
465                         f.write(ubinize_config)
466                         f.close()
467
468                         os.system("%s --compr=%s -r '%s' -o '%s.ubifs' -m %d -e %d -c %d" % (self.mkfs_ubifs, self.flash_compression, basepath, basepath, self.ubifs_min_io_size, self.ubifs_leb_size, self.ubifs_max_leb_count))
469                         os.system("%s -m %d -p %d -o '%s.ubi' '%s/ubinize.cfg'" % (self.ubinize, self.ubifs_min_io_size, self.ubinize_peb_size, basepath, self.builddir))
470
471                 # Build development rootfs
472                 if build_target == "all" or build_target == "devrootfs":
473                         print "Creating a rootstrap..."
474                         os.system("%s -c --one-file-system -C '%s' -f '%s-rootstrap.tar' ." % (self.tar, self.path, basepath))
475                         # override passwd/group with the modified versions
476                         os.system("%s -r -C '%s' -f '%s-rootstrap.tar' ./etc/passwd ./etc/group" % (self.tar, basepath, basepath))
477                         os.system("%s -f '%s-rootstrap.tar'" % (self.gzip, basepath))
478
479                 os.chdir(old_dir)
480
481                 print "Build finished:"
482                 for targets, suffix in ((('rootfs', 'all'), '.tar.gz'),
483                                         (('jffs2', 'all'), '.jffs2'),
484                                         (('ubi',), '.ubi'),
485                                         (('devrootfs', 'all'), '-rootstrap.tar.gz')):
486                         if build_target in targets:
487                                 fullname = basepath + suffix
488                                 s = os.stat(fullname)
489                                 size = s.st_size / 1024.0 / 1024.0
490                                 print '  %-40s %.2f MiB' % (fullname, size)
491
492                 if clean:
493                         print "Cleaning up"
494                         os.remove("/tmp/env.faked")
495                         os.system("rm -rf '%s'" % basepath)