Scons cython builder (#316)

* migrate to home pc

* style

* this works

* this is hacky

* why doesn't this work

* this should work

* no lint

* link against libdbc wtih abs path

* new env

Co-authored-by: Adeeb Shihadeh <adeebshihadeh@gmail.com>
This commit is contained in:
grekiki 2020-11-27 01:11:23 +01:00 committed by GitHub
parent c072d9e8df
commit 699dc63a88
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 85 additions and 118 deletions

1
.gitignore vendored
View File

@ -1,6 +1,7 @@
.mypy_cache/
*.pyc
*.os
*.o
*.tmp
*.dylib
.*.swp

View File

@ -14,6 +14,7 @@ repos:
rev: master
hooks:
- id: flake8
exclude: 'site_scons/'
args:
- --ignore=E111,E114,E121,E124,E302,E501,E741
- --statistics
@ -24,6 +25,7 @@ repos:
entry: pylint
language: system
types: [python]
exclude: 'site_scons/'
args:
- --disable=C,R,W0613,W0511,W0212,W0201,W0311,W0106,W0603,W0621,W0703,E1136
- repo: local

View File

@ -46,6 +46,7 @@ WORKDIR /project
RUN git clone https://github.com/commaai/cereal.git /project/cereal
COPY SConstruct .
COPY ./site_scons /project/site_scons
COPY . /project/opendbc
RUN rm -rf /project/opendbc/.git

View File

@ -1,24 +1,20 @@
import Cython
import distutils
import os
import subprocess
import sys
import sysconfig
zmq = 'zmq'
arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
# Rebuild cython extensions if python, distutils, or cython change
cython_dependencies = [Value(v) for v in (sys.version, distutils.__version__, Cython.__version__)]
Export('cython_dependencies')
cereal_dir = Dir('.')
python_path = sysconfig.get_paths()['include']
cpppath = [
'#',
'#cereal',
"#cereal/messaging",
"#opendbc/can",
'/usr/lib/include',
'#',
'#cereal',
"#cereal/messaging",
"#opendbc/can",
'/usr/lib/include',
python_path
]
AddOption('--test',
@ -45,10 +41,14 @@ env = Environment(
] + ccflags_asan,
LDFLAGS=ldflags_asan,
LINKFLAGS=ldflags_asan,
LIBPATH=[
"#opendbc/can/",
],
CFLAGS="-std=gnu11",
CXXFLAGS="-std=c++1z",
CPPPATH=cpppath,
CYTHONCFILESUFFIX=".cpp",
tools=["default", "cython"]
)
Export('env', 'zmq', 'arch')
@ -57,5 +57,24 @@ cereal = [File('#cereal/libcereal.a')]
messaging = [File('#cereal/libmessaging.a')]
Export('cereal', 'messaging')
envCython = env.Clone()
envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-deprecated-declarations"]
python_libs = []
if arch == "Darwin":
envCython["LINKFLAGS"] = ["-bundle", "-undefined", "dynamic_lookup"]
elif arch == "aarch64":
envCython["LINKFLAGS"] = ["-shared"]
python_libs.append(os.path.basename(python_path))
else:
envCython["LINKFLAGS"] = ["-pthread", "-shared"]
envCython["LIBS"] = python_libs
Export('envCython')
SConscript(['cereal/SConscript'])
SConscript(['opendbc/can/SConscript'])

View File

@ -1,4 +1,4 @@
Import('env', 'cereal', 'cython_dependencies')
Import('env', 'envCython', 'cereal')
import os
from opendbc.can.process_dbc import process
@ -13,10 +13,10 @@ for x in sorted(os.listdir('../')):
dbc = env.Command(out_fn, in_fn, compile_dbc)
dbcs.append(dbc)
libdbc = env.SharedLibrary('libdbc', ["dbc.cc", "parser.cc", "packer.cc", "common.cc"]+dbcs, LIBS=["capnp", "kj"])
# Build packer and parser
env.Command(['packer_pyx.so', 'packer_pyx.cpp', 'parser_pyx.so', 'parser_pyx.cpp'],
cython_dependencies + [libdbc, cereal, 'common_pyx_setup.py', 'common.pxd', 'packer_pyx.pyx', 'parser_pyx.pyx', 'packer.cc', 'parser.cc'],
"cd opendbc/can && python3 common_pyx_setup.py build_ext --inplace")
lenv = envCython.Clone()
lenv["LINKFLAGS"] += [libdbc[0].get_labspath()]
lenv.Program('parser_pyx.so', 'parser_pyx.pyx')
lenv.Program('packer_pyx.so', 'packer_pyx.pyx')

View File

@ -1,94 +0,0 @@
import os
import subprocess
import sysconfig
import platform
from distutils.core import Extension, setup # pylint: disable=import-error,no-name-in-module
from Cython.Build import cythonize
from Cython.Distutils import build_ext
ANNOTATE = os.getenv('ANNOTATE') is not None
BASEDIR = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../"))
def get_ext_filename_without_platform_suffix(filename):
name, ext = os.path.splitext(filename)
ext_suffix = sysconfig.get_config_var('EXT_SUFFIX')
if ext_suffix == ext:
return filename
ext_suffix = ext_suffix.replace(ext, '')
idx = name.find(ext_suffix)
if idx == -1:
return filename
else:
return name[:idx] + ext
class BuildExtWithoutPlatformSuffix(build_ext):
def get_ext_filename(self, ext_name):
filename = super().get_ext_filename(ext_name)
return get_ext_filename_without_platform_suffix(filename)
extra_compile_args = ["-std=c++1z", "-Wno-nullability-completeness"]
ARCH = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip() # pylint: disable=unexpected-keyword-arg
if ARCH == "aarch64":
extra_compile_args += ["-Wno-deprecated-register"]
if platform.system() == "Darwin":
libdbc = "libdbc.dylib"
else:
libdbc = "libdbc.so"
extra_link_args = [os.path.join(BASEDIR, 'opendbc', 'can', libdbc)]
include_dirs = [
BASEDIR,
os.path.join(BASEDIR, 'phonelibs'),
]
# Build CAN Parser
setup(name='CAN parser',
cmdclass={'build_ext': BuildExtWithoutPlatformSuffix},
ext_modules=cythonize(
Extension(
"parser_pyx",
language="c++",
sources=['parser_pyx.pyx'],
extra_compile_args=extra_compile_args,
include_dirs=include_dirs,
extra_link_args=extra_link_args,
),
nthreads=4,
annotate=ANNOTATE
),
)
if platform.system() == "Darwin":
os.system("install_name_tool -change opendbc/can/libdbc.dylib " + BASEDIR + "/opendbc/can/libdbc.dylib parser_pyx.so")
# Build CAN Packer
setup(name='CAN packer',
cmdclass={'build_ext': BuildExtWithoutPlatformSuffix},
ext_modules=cythonize(
Extension(
"packer_pyx",
language="c++",
sources=['packer_pyx.pyx'],
extra_compile_args=extra_compile_args,
include_dirs=include_dirs,
extra_link_args=extra_link_args,
),
nthreads=4,
annotate=ANNOTATE
),
)
if platform.system() == "Darwin":
os.system("install_name_tool -change opendbc/can/libdbc.dylib " + BASEDIR + "/opendbc/can/libdbc.dylib packer_pyx.so")

View File

@ -8,8 +8,8 @@ from libcpp.string cimport string
from libcpp cimport bool
from posix.dlfcn cimport dlopen, dlsym, RTLD_LAZY
from common cimport CANPacker as cpp_CANPacker
from common cimport dbc_lookup, SignalPackValue, DBC
from .common cimport CANPacker as cpp_CANPacker
from .common cimport dbc_lookup, SignalPackValue, DBC
cdef class CANPacker:
@ -22,8 +22,8 @@ cdef class CANPacker:
def __init__(self, dbc_name):
self.dbc = dbc_lookup(dbc_name)
if not self.dbc:
raise RuntimeError("Can't lookup" + dbc_name)
raise RuntimeError(f"Can't lookup {dbc_name}")
self.packer = new cpp_CANPacker(dbc_name)
num_msgs = self.dbc[0].num_msgs
for i in range(num_msgs):

View File

@ -8,8 +8,8 @@ from libc.stdint cimport uint32_t, uint64_t, uint16_t
from libcpp.map cimport map
from libcpp cimport bool
from common cimport CANParser as cpp_CANParser
from common cimport SignalParseOptions, MessageParseOptions, dbc_lookup, SignalValue, DBC
from .common cimport CANParser as cpp_CANParser
from .common cimport SignalParseOptions, MessageParseOptions, dbc_lookup, SignalValue, DBC
import os
import numbers

View File

@ -0,0 +1,38 @@
import SCons
from SCons.Action import Action
cythonAction = Action("$CYTHONCOM")
def create_builder(env):
try:
cython = env['BUILDERS']['Cython']
except KeyError:
cython = SCons.Builder.Builder(
action = cythonAction,
emitter = {},
suffix = cython_suffix_emitter,
single_source = 1
)
env['BUILDERS']['Cython'] = cython
return cython
def cython_suffix_emitter(env, source):
return "$CYTHONCFILESUFFIX"
def generate(env):
env["CYTHON"] = "cythonize"
env["CYTHONCOM"] = "$CYTHON $CYTHONFLAGS $SOURCE"
env["CYTHONCFILESUFFIX"] = ".cpp"
c_file, _ = SCons.Tool.createCFileBuilders(env)
c_file.suffix['.pyx'] = cython_suffix_emitter
c_file.add_action('.pyx', cythonAction)
c_file.suffix['.py'] = cython_suffix_emitter
c_file.add_action('.py', cythonAction)
create_builder(env)
def exists(env):
return True