rootfs.py: Added support for creating UBI images.
[matrix.git] / matrix / rootfs.py
1 # Copyright (C) 2007-2008 Movial Oy
2 # Authors: Kalle Vahlman <kalle.vahlman@movial.fi>
3 #          Timo Savola <tsavola@movial.fi>
4 #          Tuomas Kulve <tuomas.kulve@movial.fi>
5
6 import errno
7 import optparse
8 import os
9 import re
10 import shutil
11 import sys
12
13 from config import config
14 from config import parse as config_parse
15
16 image_type_list = (
17         'jffs2',
18         'ubi',
19 )
20 image_type_default = 'jffs2'
21
22 def main():
23         parser = optparse.OptionParser(usage='%prog [<options>]')
24
25         parser.add_option(
26                 '-n', '--no-clean',
27                 help='do not clean env.faked and stripped directory',
28                 dest='clean', action='store_false', default=True)
29
30         parser.add_option(
31                 '-i', '--image-only',
32                 help='generate only a filesystem image',
33                 dest='image_only', action='store_true')
34
35         parser.add_option(
36                 '-T', '--image-type',
37                 help='type of generated filesystem image: %s [default: %s]' % \
38                      (', '.join(image_type_list), image_type_default),
39                 dest='image_type', metavar='TYPE',
40                 choices=image_type_list, default=image_type_default)
41
42         parser.add_option(
43                 '-r', '--rootfs-only',
44                 help='generate only a stripped archive',
45                 dest='rootfs_only', action='store_true')
46
47         parser.add_option(
48                 '-d', '--devrootfs-only',
49                 help='generate only an archive with development files',
50                 dest='devrootfs_only', action='store_true')
51
52         opts, args = parser.parse_args()
53         if args:
54                 parser.print_help()
55                 sys.exit(1)
56
57         config_parse('config.local')
58         config_parse('config')
59         config_parse('boards')
60         config_parse('rootfs')
61
62         f = os.popen('sb-conf cu')
63         target = f.read()
64         f.close()
65
66         build = Builder(target.strip())
67         build.include_paths(config.include_paths)
68         build.include_files(config.include_files)
69         build.filter_paths(config.exclude_paths)
70         build.filter_files(config.exclude_files)
71         build.filter_expressions(config.exclude_expressions)
72         build.add_paths(config.created_paths)
73         build.set_devices(config.devices)
74         build.set_change_owner(config.change_owner)
75         build.set_flash_erase_size(config.boards[config.board].flash_erase_size)
76         build.set_flash_pad_size(config.boards[config.board].flash_pad_size)
77
78         build.set_flash_compression(config.boards[config.board].flash_compression)
79         build.set_ubifs_leb_size(config.boards[config.board].ubifs_leb_size)
80         build.set_ubifs_max_leb_count(config.boards[config.board].ubifs_max_leb_count)
81         build.set_ubifs_min_io_size(config.boards[config.board].ubifs_min_io_size)
82         build.set_ubinize_config_vol_size(config.boards[config.board].ubinize_config_vol_size)
83
84         if opts.rootfs_only:
85                 build.generate(opts.clean, build_target="rootfs")
86         elif opts.image_only:
87                 build.generate(opts.clean, build_target=opts.image_type)
88         elif opts.devrootfs_only:
89                 build.generate(opts.clean, build_target="devrootfs")
90         else:
91                 build.generate(opts.clean, build_target="all")
92
93 class Builder(object):
94         builddir = "/tmp"
95         target = None
96
97         # Tools
98         strip = "strip"
99         mknod = "fakeroot -i /tmp/env.faked -s /tmp/env.faked mknod"
100         chmod = "fakeroot -i /tmp/env.faked -s /tmp/env.faked chmod"
101         chown = "fakeroot -i /tmp/env.faked -s /tmp/env.faked chown"
102         mkfs_jffs2 = "fakeroot -i /tmp/env.faked -s /tmp/env.faked mkfs.jffs2"
103         mkfs_ubifs = "fakeroot -i /tmp/env.faked -s /tmp/env.faked mkfs.ubifs"
104         ubinize = "fakeroot -i /tmp/env.faked -s /tmp/env.faked ubinize"
105         tar = "fakeroot -i /tmp/env.faked -s /tmp/env.faked tar"
106         flash_erase_size = 0x4000
107         flash_pad_size = 0x3e00000
108         # Compression to use
109         flash_compression = "lzo"
110         # Logical erase block size
111         ubifs_leb_size = 130944
112         # Maximum logical erase block count
113         ubifs_max_leb_count = 120
114         # Minimum I/O unit size
115         ubifs_min_io_size = 1
116         # Size of the physical eraseblock of the flash
117         ubinize_peb_size = flash_erase_size
118
119         # Configation values for ubinize
120         ubinize_config_mode = "ubi"
121         ubinize_config_image = None
122         ubinize_config_vol_id = 0
123         ubinize_config_vol_size = 13631488
124         ubinize_config_vol_type = "dynamic"
125         ubinize_config_vol_name = "rootfs"
126         ubinize_config_vol_aligment = 1
127         ubinize_config_vol_flags = "autoresize"
128         
129
130         file_list = []
131         remove_list = []
132
133         created_paths = []
134         devices = {}
135         chown_owner = {}
136
137         def __init__(self, mytarget):
138                 self.target = mytarget
139
140         def include_paths(self, paths):
141                 old_dir = os.getcwd()
142                 os.chdir("/targets")
143
144                 # Traverse include_paths hierarchies and grab all files there
145                 for path in paths:
146                         for root, dirs, files in os.walk(self.target + path):
147                                 for f in files:
148                                         self.file_list.append("%s/%s" % (root, f))
149                                 if len(files) == 0:
150                                         self.file_list.append(root)
151
152                 os.chdir(old_dir)
153
154         def include_files(self, files):
155                 old_dir = os.getcwd()
156                 os.chdir("/targets")
157
158                 # Append included individual files
159                 for f in files:
160                         self.file_list.append(self.target + f)
161
162                 os.chdir(old_dir)
163
164         def filter_paths(self, paths):
165                 # Filter out files in excluded paths
166                 for f in self.file_list:
167                         for d in paths:
168                                 if f == self.target + d or f.startswith(self.target + d + '/'):
169                                         self.remove_list.append(f)
170
171         def filter_files(self, files):
172                 # Filter out excluded files
173                 for f in files:
174                         self.remove_list.append(self.target + f)
175
176         def filter_expressions(self, expressions):
177                 # Filter out files matching excluded patterns
178                 compiled_patterns = []
179                 for pattern_str in expressions:
180                         compiled_patterns.append(re.compile(pattern_str))
181
182                 for f in self.file_list:
183                         for pattern in compiled_patterns:
184                                 if pattern.search(f):
185                                         self.remove_list.append(f)
186
187         def add_paths(self, paths):
188                 for path in paths:
189                         self.created_paths.append(path)
190
191         def set_devices(self, devices):
192                 if devices is not None:
193                         self.devices = devices
194
195         def set_change_owner(self, change_owner):
196                 if change_owner is not None:
197                         self.change_owner= change_owner
198
199         def set_flash_erase_size(self, size):
200                 if size is not None:
201                         self.flash_erase_size = size
202                         self.ubinize_peb_size = size
203
204         def set_flash_pad_size(self, size):
205                 if size is not None:
206                         self.flash_pad_size = size
207
208         def set_flash_compression(self, compr):
209                 if compr is not None:
210                         self.flash_compression = compr
211
212         def set_ubifs_leb_size(self, size):
213                 if size is not None:
214                         self.ubifs_leb_size = size
215
216         def set_ubifs_max_leb_count(self, count):
217                 if count is not None:
218                         self.ubifs_max_leb_count = count
219
220         def set_ubifs_min_io_size(self, size):
221                 if size is not None:
222                         self.ubifs_min_io_size = size
223
224         def set_ubinize_config_vol_size(self, size):
225                 if size is not None:
226                         self.ubinize_config_vol_size = size
227
228         def generate(self, clean, build_target="all"):
229                 copy_list = self.file_list
230
231                 for f in self.remove_list:
232                                 try:
233                                         copy_list.remove(f)
234                                 except:
235                                         pass
236
237                 n_files = len(copy_list)*1.0
238                 current_percent = 0.0
239                 current_file = 0.0
240
241                 old_dir = os.getcwd()
242                 os.chdir("/targets")
243
244                 if os.path.exists(os.path.join(self.builddir, self.target)):
245                         print "%s exists, please remove it and try again" % os.path.join(self.builddir, self.target)
246                         return
247
248                 # Copy the wanted files, strip binaries
249                 print "Copying files..."
250                 for f in copy_list:
251                         if f[0] == '/':
252                                 f = f[1:]
253                         path = os.path.join(self.builddir, os.path.dirname(f))
254                         try:
255                                 os.makedirs(path)
256                         except OSError, e:
257                                 if e.errno == errno.EEXIST:
258                                         pass
259
260                         if os.path.islink(f):
261                                 link_dest = os.readlink(f)
262                                 os.symlink(link_dest, os.path.join(self.builddir, f))
263                         elif os.path.isdir(f):
264                                 try:
265                                         os.makedirs(os.path.join(self.builddir, f))
266                                 except OSError, e:
267                                         if e.errno == errno.EEXIST:
268                                                 pass
269                         else:
270                                 shutil.copy(f, path)
271                                 if not f.endswith(".ko"):
272                                         os.system(self.strip + " " + os.path.join(path,os.path.basename(f)) + " &> /dev/null")
273
274                         if (current_file/n_files)*100 > current_percent:
275                                 current_percent += 10.0
276                                 print "%0.1f%%" % current_percent,
277                                 sys.stdout.flush()
278
279                         current_file += 1.0
280
281                 # Create extra directory structure
282                 print "\nCreating extra directories..."
283                 for d in self.created_paths:
284                         if d[0] == '/':
285                                 d = d[1:]
286                         path = os.path.join(self.builddir, self.target, d)
287                         try:
288                                 os.makedirs(path)
289                         except OSError, e:
290                                 if e.errno == errno.EEXIST:
291                                         pass
292
293                 # Remove always the fakeroot environment.
294                 if os.path.exists("/tmp/env.faked"):
295                         os.remove("/tmp/env.faked")
296                 os.system("touch /tmp/env.faked")
297
298                 # Create device nodes
299                 print "Creating device nodes..."
300                 for d in self.devices.keys():
301                         mode = "664"
302                         if len(self.devices[d]) == 3:
303                                 device_type,major,minor = self.devices[d]
304                         else:
305                                 device_type,major,minor,mode = self.devices[d]
306                         os.system("%s -m %s %s/%s/dev/%s %s %i %i" % (self.mknod, mode, self.builddir, self.target, d, device_type, major, minor))
307
308                 # Fix permissions
309                 print "Adjusting ownerships and permissions..."
310                 # Make the whole fs root-owned
311                 os.system("%s root.root -R %s/%s" % (self.chown, self.builddir, self.target))
312
313                 # For sshd:
314                 os.system("%s go-r %s/%s/etc/ssh*key*" % (self.chmod, self.builddir, self.target))
315
316                 # For /var/empty (sshd requires this):
317                 os.system("%s go-rw /%s/%s/var/empty" % (self.chmod, self.builddir, self.target))
318
319                 # /dev/null needs to be writable by all:
320                 os.system("%s a+rw /%s/%s/dev/null" % (self.chmod, self.builddir, self.target))
321
322                 # Set non-root owners
323                 print "Setting non-root ownerships..."
324                 for d in self.change_owner.keys():
325                         dir = d
326                         if dir[0] == '/':
327                                 dir = dir[1:]
328                         if os.path.exists(os.path.join(self.builddir, self.target, dir)):
329                                 user,group = self.change_owner[d]
330                                 os.system("%s %s.%s -R %s/%s/%s " % (self.chown, user, group, self.builddir, self.target, dir))
331
332                 # Build rootfs
333                 if build_target == "all" or build_target == "rootfs":
334                         print "Creating a rootfs..."
335                         os.system("%s -c --one-file-system -C %s/%s -z -f %s/%s.tgz ." % (self.tar, self.builddir, self.target, self.builddir, self.target))
336
337                 # Build jffs2
338                 if build_target == "all" or build_target == "jffs2":
339                         print "Creating a root image as JFFS2..."
340                         os.system("%s -p%d -n -e%d -r %s/%s -o %s/%s.jffs2" % (self.mkfs_jffs2, self.flash_pad_size, self.flash_erase_size, self.builddir, self.target, self.builddir, self.target))
341
342                 # Build ubifs
343                 if build_target == "ubi":
344                         print "Creating a root image as UBI..."
345                         ubinize_config = ""
346                         ubinize_config += "[ubifs]\n"
347                         ubinize_config += "mode=%s\n" % self.ubinize_config_mode
348                         ubinize_config += "image=%s/%s.ubifs\n" % (self.builddir, self.target)
349                         ubinize_config += "vol_id=%d\n" % self.ubinize_config_vol_id
350                         ubinize_config += "vol_size=%d\n" % self.ubinize_config_vol_size
351                         ubinize_config += "vol_type=%s\n" % self.ubinize_config_vol_type
352                         ubinize_config += "vol_name=%s\n" % self.ubinize_config_vol_name
353                         ubinize_config += "vol_alignment=%d\n" % self.ubinize_config_vol_aligment
354                         ubinize_config += "vol_flags=%s\n" % self.ubinize_config_vol_flags
355                         f=open('/tmp/ubinize.cfg', 'w')
356                         f.write(ubinize_config)
357                         f.close()
358
359                         os.system("%s --compr=%s -r %s/%s -o %s/%s.ubifs -m %d -e %d -c %d" % (self.mkfs_ubifs, self.flash_compression, self.builddir, self.target, self.builddir, self.target, self.ubifs_min_io_size, self.ubifs_leb_size, self.ubifs_max_leb_count))
360                         os.system("%s -m %d -p %d -o %s/%s.ubi %s/ubinize.cfg" % (self.ubinize, self.ubifs_min_io_size, self.ubinize_peb_size, self.builddir, self.target, self.builddir))
361
362                 # Build development rootfs
363                 if build_target == "all" or build_target == "devrootfs":
364                         print "Creating a rootstrap..."
365                         os.system("%s -c --one-file-system -C /targets/%s -z -f %s/%s-rootstrap.tgz ." % ("tar", self.target, self.builddir, self.target))
366
367                 os.chdir(old_dir)
368
369                 print "Build finished:"
370                 if build_target == "all" or build_target == "rootfs":
371                         s = os.stat(self.builddir + "/" + self.target + ".tgz");
372                         print "\t%s/%s.tgz\t%0.2f MiB" % (self.builddir, self.target, s.st_size/1024.0/1024.0)
373                 if build_target == "all" or build_target == "jffs2":
374                         s = os.stat(self.builddir + "/" + self.target + ".jffs2");
375                         print "\t%s/%s.jffs2\t%0.2f MiB" % (self.builddir, self.target, s.st_size/1024.0/1024.0)
376                 if build_target == "ubi":
377                         s = os.stat(self.builddir + "/" + self.target + ".ubi");
378                         print "\t%s/%s.ubi\t%0.2f MiB" % (self.builddir, self.target, s.st_size/1024.0/1024.0)
379                 if build_target == "all" or build_target == "devrootfs":
380                         s = os.stat(self.builddir + "/" + self.target + "-rootstrap.tgz");
381                         print "\t%s/%s-rootstrap.tgz\t%0.2f MiB" % (self.builddir, self.target, s.st_size/1024.0/1024.0)
382
383                 if clean:
384                         print "Cleaning up"
385                         os.remove("/tmp/env.faked")
386                         os.system("rm -rf %s/%s" % (self.builddir, self.target))