install-sysroot.py 7.34 KB
Newer Older
1 2 3 4 5
#!/usr/bin/env python
# Copyright (c) 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

6
"""Install Debian sysroots for building chromium.
7 8
"""

9 10 11 12 13 14 15 16 17 18 19
# The sysroot is needed to ensure that binaries that get built will run on
# the oldest stable version of Debian that we currently support.
# This script can be run manually but is more often run as part of gclient
# hooks. When run from hooks this script is a no-op on non-linux platforms.

# The sysroot image could be constructed from scratch based on the current state
# of the Debian archive but for consistency we use a pre-built root image (we
# don't want upstream changes to Debian to effect the chromium build until we
# choose to pull them in). The images will normally need to be rebuilt every
# time chrome's build dependencies are changed but should also be updated
# periodically to include upstream security fixes from Debian.
20

21
import hashlib
22
import json
23 24 25 26 27 28 29
import platform
import optparse
import os
import re
import shutil
import subprocess
import sys
30
import urllib2
31 32

SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
33
sys.path.append(os.path.dirname(os.path.dirname(SCRIPT_DIR)))
34 35 36 37 38
import detect_host_arch
import gyp_chromium
import gyp_environment


39
URL_PREFIX = 'https://commondatastorage.googleapis.com'
40
URL_PATH = 'chrome-linux-sysroot/toolchain'
41

42
VALID_ARCHS = ('arm', 'arm64', 'i386', 'amd64', 'mips')
43

44

45 46 47 48
class Error(Exception):
  pass


49
def GetSha1(filename):
50 51 52 53 54 55 56 57 58 59 60
  sha1 = hashlib.sha1()
  with open(filename, 'rb') as f:
    while True:
      # Read in 1mb chunks, so it doesn't all have to be loaded into memory.
      chunk = f.read(1024*1024)
      if not chunk:
        break
      sha1.update(chunk)
  return sha1.hexdigest()


61
def DetectHostArch():
62 63 64 65 66
  # Figure out host arch using build/detect_host_arch.py and
  # set target_arch to host arch
  detected_host_arch = detect_host_arch.HostArch()
  if detected_host_arch == 'x64':
    return 'amd64'
67
  if detected_host_arch == 'ia32':
68
    return 'i386'
69
  if detected_host_arch == 'arm':
70
    return 'arm'
71
  if detected_host_arch == 'arm64':
72
    return 'arm64'
73
  if detected_host_arch == 'mips':
74
    return 'mips'
75
  if detected_host_arch == 'ppc':
76
    return 'ppc'
77
  if detected_host_arch == 's390':
78
    return 's390'
79

80
  raise Error('Unrecognized host arch: %s' % detected_host_arch)
81 82


83 84
def DetectTargetArch():
  """Attempt for determine target architecture.
85

86 87
  This works by looking for target_arch in GYP_DEFINES.
  """
88 89
  # TODO(agrieve): Make this script not depend on GYP_DEFINES so that it works
  #     with GN as well.
90 91 92
  gyp_environment.SetEnvironment()
  supplemental_includes = gyp_chromium.GetSupplementalFiles()
  gyp_defines = gyp_chromium.GetGypVars(supplemental_includes)
93 94 95
  target_arch = gyp_defines.get('target_arch')
  if target_arch == 'x64':
    return 'amd64'
96
  if target_arch == 'ia32':
97
    return 'i386'
98
  if target_arch == 'arm':
99
    return 'arm'
100
  if target_arch == 'arm64':
101
    return 'arm64'
102
  if target_arch == 'mipsel':
103 104 105 106
    return 'mips'

  return None

107

108
def InstallDefaultSysroots(host_arch):
109 110 111 112 113
  """Install the default set of sysroot images.

  This includes at least the sysroot for host architecture, and the 32-bit
  sysroot for building the v8 snapshot image.  It can also include the cross
  compile sysroot for ARM/MIPS if cross compiling environment can be detected.
114 115 116 117 118

  Another reason we're installing this by default is so that developers can
  compile and run on our supported platforms without having to worry about
  flipping things back and forth and whether the sysroots have been downloaded
  or not.
119
  """
120
  InstallDefaultSysrootForArch(host_arch)
121 122

  if host_arch == 'amd64':
123 124
    InstallDefaultSysrootForArch('i386')

125 126 127
  # If we can detect a non-standard target_arch such as ARM or MIPS,
  # then install the sysroot too.  Don't attempt to install arm64
  # since this is currently and android-only architecture.
128
  target_arch = DetectTargetArch()
129
  if target_arch and target_arch not in (host_arch, 'i386'):
130
    InstallDefaultSysrootForArch(target_arch)
131 132


sbc's avatar
sbc committed
133 134 135 136 137
def main(args):
  parser = optparse.OptionParser('usage: %prog [OPTIONS]', description=__doc__)
  parser.add_option('--running-as-hook', action='store_true',
                    default=False, help='Used when running from gclient hooks.'
                                        ' Installs default sysroot images.')
138 139
  parser.add_option('--arch', type='choice', choices=VALID_ARCHS,
                    help='Sysroot architecture: %s' % ', '.join(VALID_ARCHS))
140 141 142
  parser.add_option('--all', action='store_true',
                    help='Install all sysroot images (useful when updating the'
                         ' images)')
sbc's avatar
sbc committed
143
  options, _ = parser.parse_args(args)
144 145 146 147
  if options.running_as_hook and not sys.platform.startswith('linux'):
    return 0

  if options.running_as_hook:
148 149 150 151 152
    host_arch = DetectHostArch()
    # PPC/s390 don't use sysroot, see http://crbug.com/646169
    if host_arch in ['ppc','s390']:
      return 0
    InstallDefaultSysroots(host_arch)
153
  elif options.arch:
154
    InstallDefaultSysrootForArch(options.arch)
155 156 157 158 159 160
  elif options.all:
    for arch in VALID_ARCHS:
      InstallDefaultSysrootForArch(arch)
  else:
    print 'You much specify either --arch, --all or --running-as-hook'
    return 1
161

162
  return 0
163

164

165
def InstallDefaultSysrootForArch(target_arch):
166
  if target_arch not in VALID_ARCHS:
167
    raise Error('Unknown architecture: %s' % target_arch)
168
  InstallSysroot('Stretch', target_arch)
169

170

171
def InstallSysroot(target_platform, target_arch):
172 173
  # The sysroot directory should match the one specified in
  # build/config/sysroot.gni.
174
  # TODO(thestig) Consider putting this elsewhere to avoid having to recreate
175 176 177
  # it on every build.
  linux_dir = os.path.dirname(SCRIPT_DIR)

178 179 180 181
  sysroots_file = os.path.join(SCRIPT_DIR, 'sysroots.json')
  sysroots = json.load(open(sysroots_file))
  sysroot_key = '%s_%s' % (target_platform.lower(), target_arch)
  if sysroot_key not in sysroots:
182
    raise Error('No sysroot for: %s %s' % (target_platform, target_arch))
183
  sysroot_dict = sysroots[sysroot_key]
184 185 186 187 188
  revision = sysroot_dict['Revision']
  tarball_filename = sysroot_dict['Tarball']
  tarball_sha1sum = sysroot_dict['Sha1Sum']
  sysroot = os.path.join(linux_dir, sysroot_dict['SysrootDir'])

189
  url = '%s/%s/%s/%s' % (URL_PREFIX, URL_PATH, revision, tarball_filename)
190 191 192 193 194

  stamp = os.path.join(sysroot, '.stamp')
  if os.path.exists(stamp):
    with open(stamp) as s:
      if s.read() == url:
195
        return
196

197
  print 'Installing Debian %s %s root image: %s' % \
198
      (target_platform, target_arch, sysroot)
199 200 201 202
  if os.path.isdir(sysroot):
    shutil.rmtree(sysroot)
  os.mkdir(sysroot)
  tarball = os.path.join(sysroot, tarball_filename)
203 204 205
  print 'Downloading %s' % url
  sys.stdout.flush()
  sys.stderr.flush()
206 207
  for _ in range(3):
    try:
208 209 210
      response = urllib2.urlopen(url)
      with open(tarball, "wb") as f:
        f.write(response.read())
211 212 213 214 215
      break
    except:
      pass
  else:
    raise Error('Failed to download %s' % url)
216
  sha1sum = GetSha1(tarball)
217
  if sha1sum != tarball_sha1sum:
218 219
    raise Error('Tarball sha1sum is wrong.'
                'Expected %s, actual: %s' % (tarball_sha1sum, sha1sum))
220 221 222 223 224 225 226 227
  subprocess.check_call(['tar', 'xf', tarball, '-C', sysroot])
  os.remove(tarball)

  with open(stamp, 'w') as s:
    s.write(url)


if __name__ == '__main__':
sbc's avatar
sbc committed
228 229 230 231 232
  try:
    sys.exit(main(sys.argv[1:]))
  except Error as e:
    sys.stderr.write(str(e) + '\n')
    sys.exit(1)