457 lines
17 KiB
Python
Executable File
457 lines
17 KiB
Python
Executable File
#!/usr/bin/env python2
|
|
|
|
import sys
|
|
import os
|
|
import os.path
|
|
import re
|
|
import argparse
|
|
import datetime
|
|
import multiprocessing
|
|
import shlex
|
|
import shutil
|
|
import tempfile
|
|
import time
|
|
import subprocess
|
|
import glob
|
|
|
|
|
|
# Default flags for make
|
|
default_make_flags = ["-i", "-j" + str(multiprocessing.cpu_count())]
|
|
|
|
# Set YCM-Generator directory
|
|
# Always obtain the real path to the directory where 'config_gen.py' lives as,
|
|
# in some cases, it will be a symlink placed in '/usr/bin' (as is the case
|
|
# with the Arch Linux AUR package) and it won't
|
|
# be able to find the plugin directory.
|
|
ycm_generator_dir = os.path.dirname(os.path.realpath(__file__))
|
|
|
|
|
|
def main():
|
|
# parse command-line args
|
|
parser = argparse.ArgumentParser(description="Automatically generates config files for YouCompleteMe")
|
|
parser.add_argument("-v", "--verbose", action="store_true", help="Show output from build process")
|
|
parser.add_argument("-m", "--make", default="make", help="Use the specified executable for make.")
|
|
parser.add_argument("-c", "--compiler", help="Use the specified executable for clang. It should be the same version as the libclang used by YCM. The executable for clang++ will be inferred from this.")
|
|
parser.add_argument("-C", "--configure_opts", default="", help="Additional flags to pass to configure/cmake/etc. e.g. --configure_opts=\"--enable-FEATURE\"")
|
|
parser.add_argument("-F", "--format", choices=["ycm", "cc"], default="ycm", help="Format of output file (YouCompleteMe or color_coded). Default: ycm")
|
|
parser.add_argument("-M", "--make-flags", help="Flags to pass to make when fake-building. Default: -M=\"{}\"".format(" ".join(default_make_flags)))
|
|
parser.add_argument("-o", "--output", help="Save the config file as OUTPUT. Default: .ycm_extra_conf.py, or .color_coded if --format=cc.")
|
|
parser.add_argument("-x", "--language", choices=["c", "c++"], help="Only output flags for the given language. This defaults to whichever language has its compiler invoked the most.")
|
|
parser.add_argument("--out-of-tree", action="store_true", help="Build autotools projects out-of-tree. This is a no-op for other project types.")
|
|
parser.add_argument("--qt-version", choices=["4", "5"], default="5", help="Use the given Qt version for qmake. (Default: 5)")
|
|
parser.add_argument("-e", "--preserve-environment", action="store_true", help="Pass environment variables to build processes.")
|
|
parser.add_argument("PROJECT_DIR", help="The root directory of the project.")
|
|
args = vars(parser.parse_args())
|
|
project_dir = os.path.abspath(args["PROJECT_DIR"])
|
|
|
|
# verify that project_dir exists
|
|
if(not os.path.exists(project_dir)):
|
|
print("ERROR: '{}' does not exist".format(project_dir))
|
|
return 1
|
|
|
|
# verify the clang is installed, and infer the correct name for both the C and C++ compilers
|
|
try:
|
|
cc = args["compiler"] or "clang"
|
|
args["cc"] = subprocess.check_output(["which", cc]).strip()
|
|
except subprocess.CalledProcessError:
|
|
print("ERROR: Could not find clang at '{}'. Please make sure it is installed and is either in your path, or specified with --compiler.".format(cc))
|
|
return 1
|
|
|
|
try:
|
|
h, t = os.path.split(args["compiler"] or "clang")
|
|
cxx = os.path.join(h, t.replace("clang", "clang++"))
|
|
args["cxx"] = subprocess.check_output(["which", cxx]).strip()
|
|
except subprocess.CalledProcessError:
|
|
print("ERROR: Could not find clang++ at '{}'. Please make sure it is installed and specified appropriately.".format(cxx))
|
|
return 1
|
|
|
|
# sanity check - remove this after we add Windows support
|
|
if(sys.platform.startswith("win32")):
|
|
print("ERROR: Windows is not supported")
|
|
|
|
# prompt user to overwrite existing file (if necessary)
|
|
config_file = {
|
|
None: args["output"],
|
|
"cc": os.path.join(project_dir, ".color_coded"),
|
|
"ycm": os.path.join(project_dir, ".ycm_extra_conf.py"),
|
|
}[args["format"] if args["output"] is None else None]
|
|
|
|
if(os.path.exists(config_file)):
|
|
print("'{}' already exists. Overwrite? [y/N] ".format(config_file)),
|
|
response = sys.stdin.readline().strip().lower()
|
|
|
|
if(response != "y" and response != "yes"):
|
|
return 1
|
|
|
|
# command-line args to pass to fake_build() using kwargs
|
|
args["make_cmd"] = args.pop("make")
|
|
args["configure_opts"] = shlex.split(args["configure_opts"])
|
|
args["make_flags"] = default_make_flags if args["make_flags"] is None else shlex.split(args["make_flags"])
|
|
force_lang = args.pop("language")
|
|
output_format = args.pop("format")
|
|
del args["compiler"]
|
|
del args["output"]
|
|
del args["PROJECT_DIR"]
|
|
|
|
generate_conf = {
|
|
"ycm": generate_ycm_conf,
|
|
"cc": generate_cc_conf,
|
|
}[output_format]
|
|
|
|
# temporary files to hold build logs
|
|
with tempfile.NamedTemporaryFile(mode="rw") as c_build_log:
|
|
with tempfile.NamedTemporaryFile(mode="rw") as cxx_build_log:
|
|
# perform the actual compilation of flags
|
|
fake_build(project_dir, c_build_log.name, cxx_build_log.name, **args)
|
|
(c_count, c_skip, c_flags) = parse_flags(c_build_log)
|
|
(cxx_count, cxx_skip, cxx_flags) = parse_flags(cxx_build_log)
|
|
|
|
print("Collected {} relevant entries for C compilation ({} discarded).".format(c_count, c_skip))
|
|
print("Collected {} relevant entries for C++ compilation ({} discarded).".format(cxx_count, cxx_skip))
|
|
|
|
# select the language to compile for. If -x was used, zero all other options (so we don't need to repeat the error code)
|
|
if(force_lang == "c"):
|
|
cxx_count = 0
|
|
elif(force_lang == "c++"):
|
|
c_count = 0
|
|
|
|
if(c_count == 0 and cxx_count == 0):
|
|
print()
|
|
print("ERROR: No commands were logged to the build logs (C: {}, C++: {}).".format(c_build_log.name, cxx_build_log.name))
|
|
print("Your build system may not be compatible.")
|
|
c_build_log.delete = False
|
|
cxx_build_log.delete = False
|
|
return 3
|
|
|
|
elif(c_count > cxx_count):
|
|
lang, flags = ("c", c_flags)
|
|
else:
|
|
lang, flags = ("c++", cxx_flags)
|
|
|
|
generate_conf(["-x", lang] + flags, config_file)
|
|
print("Created {} config file with {} {} flags".format(output_format.upper(), len(flags), lang.upper()))
|
|
|
|
|
|
def fake_build(project_dir, c_build_log_path, cxx_build_log_path, verbose, make_cmd, cc, cxx, out_of_tree, configure_opts, make_flags, preserve_environment, qt_version):
|
|
'''Builds the project using the fake toolchain, to collect the compiler flags.
|
|
|
|
project_dir: the directory containing the source files
|
|
build_log_path: the file to log commands to
|
|
verbose: show the build process output
|
|
make_cmd: the path of the make executable
|
|
cc: the path of the clang executable
|
|
cxx: the path of the clang++ executable
|
|
out_of_tree: perform an out-of-tree build (autotools only)
|
|
configure_opts: additional flags for configure stage
|
|
make_flags: additional flags for make
|
|
preserve_environment: pass environment variables to build processes
|
|
qt_version: The Qt version to use when building with qmake.
|
|
'''
|
|
|
|
# TODO: add Windows support
|
|
assert(not sys.platform.startswith("win32"))
|
|
fake_path = os.path.join(ycm_generator_dir, "fake-toolchain", "Unix")
|
|
|
|
# environment variables and arguments for build process
|
|
started = time.time()
|
|
FNULL = open(os.devnull, "w")
|
|
proc_opts = {} if verbose else {
|
|
"stdin": FNULL,
|
|
"stdout": FNULL,
|
|
"stderr": FNULL
|
|
}
|
|
proc_opts["cwd"] = project_dir
|
|
|
|
if(preserve_environment):
|
|
env = os.environ
|
|
else:
|
|
# Preserve HOME, since Cmake needs it to find some packages and it's
|
|
# normally there anyway. See #26.
|
|
env = dict(map(lambda x: (x, os.environ[x]), ["HOME"]))
|
|
|
|
env["PATH"] = "{}:{}".format(fake_path, os.environ["PATH"])
|
|
env["CC"] = "clang"
|
|
env["CXX"] = "clang++"
|
|
env["YCM_CONFIG_GEN_CC_LOG"] = c_build_log_path
|
|
env["YCM_CONFIG_GEN_CXX_LOG"] = cxx_build_log_path
|
|
|
|
# used during configuration stage, so that cmake, etc. can verify what the compiler supports
|
|
env_config = env.copy()
|
|
env_config["YCM_CONFIG_GEN_CC_PASSTHROUGH"] = cc
|
|
env_config["YCM_CONFIG_GEN_CXX_PASSTHROUGH"] = cxx
|
|
|
|
# use -i (ignore errors), since the makefile may include scripts which
|
|
# depend upon the existence of various output files
|
|
make_args = [make_cmd] + make_flags
|
|
|
|
# Used for the qmake build system below
|
|
pro_files = glob.glob(os.path.join(project_dir, "*.pro"))
|
|
|
|
# sanity check - make sure the toolchain is available
|
|
assert os.path.exists(fake_path), "Could not find toolchain at '{}'".format(fake_path)
|
|
|
|
# helper function to display exact commands used
|
|
def run(cmd, *args, **kwargs):
|
|
print("$ " + " ".join(cmd))
|
|
subprocess.call(cmd, *args, **kwargs)
|
|
|
|
# execute the build system
|
|
if(os.path.exists(os.path.join(project_dir, "CMakeLists.txt"))):
|
|
# cmake
|
|
# run cmake in a temporary directory, then compile the project as usual
|
|
build_dir = tempfile.mkdtemp()
|
|
proc_opts["cwd"] = build_dir
|
|
|
|
# if the project was built in-tree, we need to hide the cache file so that cmake
|
|
# populates the build dir instead of just re-generating the existing files
|
|
cache_path = os.path.join(project_dir, "CMakeCache.txt")
|
|
|
|
if(os.path.exists(cache_path)):
|
|
fd, cache_tmp = tempfile.mkstemp()
|
|
os.close(fd)
|
|
shutil.move(cache_path, cache_tmp)
|
|
else:
|
|
cache_tmp = None
|
|
|
|
print("Running cmake in '{}'...".format(build_dir))
|
|
run(["cmake", project_dir] + configure_opts, env=env_config, **proc_opts)
|
|
|
|
print("\nRunning make...")
|
|
run(make_args, env=env, **proc_opts)
|
|
|
|
print("\nCleaning up...")
|
|
print("")
|
|
shutil.rmtree(build_dir)
|
|
|
|
if(cache_tmp):
|
|
shutil.move(cache_tmp, cache_path)
|
|
|
|
elif(os.path.exists(os.path.join(project_dir, "configure"))):
|
|
# autotools
|
|
# perform build in-tree, since not all projects handle out-of-tree builds correctly
|
|
|
|
if(out_of_tree):
|
|
build_dir = tempfile.mkdtemp()
|
|
proc_opts["cwd"] = build_dir
|
|
print("Configuring autotools in '{}'...".format(build_dir))
|
|
else:
|
|
print("Configuring autotools...")
|
|
|
|
run([os.path.join(project_dir, "configure")] + configure_opts, env=env_config, **proc_opts)
|
|
|
|
print("\nRunning make...")
|
|
run(make_args, env=env, **proc_opts)
|
|
|
|
print("\nCleaning up...")
|
|
|
|
if(out_of_tree):
|
|
print("")
|
|
shutil.rmtree(build_dir)
|
|
else:
|
|
run([make_cmd, "maintainer-clean"], env=env, **proc_opts)
|
|
|
|
elif(pro_files):
|
|
# qmake
|
|
# make sure there is only one .pro file
|
|
if len(pro_files) != 1:
|
|
print("ERROR: Found {} .pro files (expected one): {}.".format(
|
|
len(pro_files), ', '.join(pro_files)))
|
|
sys.exit(1)
|
|
|
|
# run qmake in a temporary directory, then compile the project as usual
|
|
build_dir = tempfile.mkdtemp()
|
|
proc_opts["cwd"] = build_dir
|
|
env_config["QT_SELECT"] = qt_version
|
|
env_config["QMAKESPEC"] = "unsupported/linux-clang" if qt_version == "4" else "linux-clang"
|
|
|
|
print("Running qmake in '{}' with Qt {}...".format(build_dir, qt_version))
|
|
run(["qmake"] + configure_opts + [pro_files[0]], env=env_config,
|
|
**proc_opts)
|
|
|
|
print("\nRunning make...")
|
|
run(make_args, env=env, **proc_opts)
|
|
|
|
print("\nCleaning up...")
|
|
print("")
|
|
shutil.rmtree(build_dir)
|
|
|
|
elif(any([os.path.exists(os.path.join(project_dir, x)) for x in ["GNUmakefile", "makefile", "Makefile"]])):
|
|
# make
|
|
# needs to be handled last, since other build systems can generate Makefiles
|
|
print("Preparing build directory...")
|
|
run([make_cmd, "clean"], env=env, **proc_opts)
|
|
|
|
print("\nRunning make...")
|
|
run(make_args, env=env, **proc_opts)
|
|
|
|
else:
|
|
print("ERROR: Unknown build system")
|
|
sys.exit(2)
|
|
|
|
print("Build completed in {} sec".format(round(time.time() - started, 2)))
|
|
print("")
|
|
|
|
|
|
def parse_flags(build_log):
|
|
'''Creates a list of compiler flags from the build log.
|
|
|
|
build_log: an iterator of lines
|
|
Returns: (line_count, skip_count, flags)
|
|
flags is a list, and the counts are integers
|
|
'''
|
|
|
|
# Used to ignore entries which result in temporary files, or don't fully
|
|
# compile the file
|
|
temp_output = re.compile("(-x assembler)|(-o ([a-zA-Z0-9._].tmp))|(/dev/null)")
|
|
skip_count = 0
|
|
|
|
# Flags we want:
|
|
# -includes (-i, -I)
|
|
# -defines (-D)
|
|
# -warnings (-Werror), but no assembler, etc. flags (-Wa,-option)
|
|
# -language (-std=gnu99) and standard library (-nostdlib)
|
|
# -word size (-m64)
|
|
flags_whitelist = ["-[iID].*", "-W[^,]*", "-std=[a-z0-9+]+", "-(no)?std(lib|inc)", "-m[0-9]+"]
|
|
flags_whitelist = re.compile("|".join(map("^{}$".format, flags_whitelist)))
|
|
flags = set()
|
|
line_count = 0
|
|
|
|
# macro definitions should be handled separately, so we can resolve duplicates
|
|
define_flags = dict()
|
|
define_regex = re.compile("-D([a-zA-Z0-9_]+)=(.*)")
|
|
|
|
# Used to only bundle filenames with applicable arguments
|
|
filename_flags = ["-o", "-I", "-isystem", "-include", "-imacros"]
|
|
|
|
# Process build log
|
|
for line in build_log:
|
|
if(temp_output.search(line)):
|
|
skip_count += 1
|
|
continue
|
|
|
|
line_count += 1
|
|
words = split_flags(line)
|
|
|
|
for (i, word) in enumerate(words):
|
|
if(word[0] != '-' or not flags_whitelist.match(word)):
|
|
continue
|
|
|
|
# handle macro definitions
|
|
m = define_regex.match(word)
|
|
if(m):
|
|
if(m.group(1) not in define_flags):
|
|
define_flags[m.group(1)] = [m.group(2)]
|
|
elif(m.group(2) not in define_flags[m.group(1)]):
|
|
define_flags[m.group(1)].append(m.group(2))
|
|
|
|
continue
|
|
|
|
# include arguments for this option, if there are any, as a tuple
|
|
if(i != len(words) - 1 and word in filename_flags and words[i + 1][0] != '-'):
|
|
flags.add((word, words[i + 1]))
|
|
else:
|
|
flags.add(word)
|
|
|
|
# Only specify one word size (the largest)
|
|
# (Different sizes are used for different files in the linux kernel.)
|
|
mRegex = re.compile("^-m[0-9]+$")
|
|
word_flags = list([f for f in flags if isinstance(f, basestring) and mRegex.match(f)])
|
|
|
|
if(len(word_flags) > 1):
|
|
for flag in word_flags:
|
|
flags.remove(flag)
|
|
|
|
flags.add(max(word_flags))
|
|
|
|
# Resolve duplicate macro definitions (always choose the last value for consistency)
|
|
for name, values in define_flags.iteritems():
|
|
if(len(values) > 1):
|
|
print("WARNING: {} distinct definitions of macro {} found".format(len(values), name))
|
|
values.sort()
|
|
|
|
flags.add("-D{}={}".format(name, values[0]))
|
|
|
|
return (line_count, skip_count, sorted(flags))
|
|
|
|
|
|
def generate_cc_conf(flags, config_file):
|
|
'''Generates the .color_coded file
|
|
|
|
flags: the list of flags
|
|
config_file: the path to save the configuration file at'''
|
|
|
|
with open(config_file, "w") as output:
|
|
for flag in flags:
|
|
if(isinstance(flag, basestring)):
|
|
output.write(flag + "\n")
|
|
else: # is tuple
|
|
for f in flag:
|
|
output.write(f + "\n")
|
|
|
|
|
|
def generate_ycm_conf(flags, config_file):
|
|
'''Generates the .ycm_extra_conf.py.
|
|
|
|
flags: the list of flags
|
|
config_file: the path to save the configuration file at'''
|
|
|
|
template_file = os.path.join(ycm_generator_dir, "template.py")
|
|
|
|
with open(template_file, "r") as template:
|
|
with open(config_file, "w") as output:
|
|
output.write("# Generated by YCM Generator at {}\n\n".format(str(datetime.datetime.today())))
|
|
|
|
for line in template:
|
|
if(line == " # INSERT FLAGS HERE\n"):
|
|
# insert generated code
|
|
for flag in flags:
|
|
if(isinstance(flag, basestring)):
|
|
output.write(" '{}',\n".format(flag))
|
|
else: # is tuple
|
|
output.write(" '{}', '{}',\n".format(*flag))
|
|
|
|
else:
|
|
# copy template
|
|
output.write(line)
|
|
|
|
|
|
def split_flags(line):
|
|
'''Helper method that splits a string into flags.
|
|
Flags are space-seperated, except for spaces enclosed in quotes.
|
|
Returns a list of flags'''
|
|
|
|
# Pass 1: split line using whitespace
|
|
words = line.strip().split()
|
|
|
|
# Pass 2: merge words so that the no. of quotes is balanced
|
|
res = []
|
|
|
|
for w in words:
|
|
if(len(res) > 0 and unbalanced_quotes(res[-1])):
|
|
res[-1] += " " + w
|
|
else:
|
|
res.append(w)
|
|
|
|
return res
|
|
|
|
|
|
def unbalanced_quotes(s):
|
|
'''Helper method that returns True if the no. of single or double quotes in s is odd.'''
|
|
|
|
single = 0
|
|
double = 0
|
|
|
|
for c in s:
|
|
if(c == "'"):
|
|
single += 1
|
|
elif(c == '"'):
|
|
double += 1
|
|
|
|
return (single % 2 == 1 or double % 2 == 1)
|
|
|
|
|
|
if(__name__ == "__main__"):
|
|
# Note that sys.exit() lets us use None and 0 interchangably
|
|
sys.exit(main())
|
|
|