mirror of https://github.com/rusefi/opendbc.git
Scons cython builder (#316)
* migrate to home pc * style * this works * this is hacky * why doesn't this work * this should work * no lint * link against libdbc wtih abs path * new env Co-authored-by: Adeeb Shihadeh <adeebshihadeh@gmail.com>
This commit is contained in:
parent
c072d9e8df
commit
699dc63a88
|
@ -1,6 +1,7 @@
|
||||||
.mypy_cache/
|
.mypy_cache/
|
||||||
*.pyc
|
*.pyc
|
||||||
*.os
|
*.os
|
||||||
|
*.o
|
||||||
*.tmp
|
*.tmp
|
||||||
*.dylib
|
*.dylib
|
||||||
.*.swp
|
.*.swp
|
||||||
|
|
|
@ -14,6 +14,7 @@ repos:
|
||||||
rev: master
|
rev: master
|
||||||
hooks:
|
hooks:
|
||||||
- id: flake8
|
- id: flake8
|
||||||
|
exclude: 'site_scons/'
|
||||||
args:
|
args:
|
||||||
- --ignore=E111,E114,E121,E124,E302,E501,E741
|
- --ignore=E111,E114,E121,E124,E302,E501,E741
|
||||||
- --statistics
|
- --statistics
|
||||||
|
@ -24,6 +25,7 @@ repos:
|
||||||
entry: pylint
|
entry: pylint
|
||||||
language: system
|
language: system
|
||||||
types: [python]
|
types: [python]
|
||||||
|
exclude: 'site_scons/'
|
||||||
args:
|
args:
|
||||||
- --disable=C,R,W0613,W0511,W0212,W0201,W0311,W0106,W0603,W0621,W0703,E1136
|
- --disable=C,R,W0613,W0511,W0212,W0201,W0311,W0106,W0603,W0621,W0703,E1136
|
||||||
- repo: local
|
- repo: local
|
||||||
|
|
|
@ -46,6 +46,7 @@ WORKDIR /project
|
||||||
RUN git clone https://github.com/commaai/cereal.git /project/cereal
|
RUN git clone https://github.com/commaai/cereal.git /project/cereal
|
||||||
|
|
||||||
COPY SConstruct .
|
COPY SConstruct .
|
||||||
|
COPY ./site_scons /project/site_scons
|
||||||
COPY . /project/opendbc
|
COPY . /project/opendbc
|
||||||
|
|
||||||
RUN rm -rf /project/opendbc/.git
|
RUN rm -rf /project/opendbc/.git
|
||||||
|
|
45
SConstruct
45
SConstruct
|
@ -1,24 +1,20 @@
|
||||||
import Cython
|
|
||||||
import distutils
|
|
||||||
import os
|
import os
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sysconfig
|
||||||
|
|
||||||
zmq = 'zmq'
|
zmq = 'zmq'
|
||||||
arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
|
arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
|
||||||
|
|
||||||
# Rebuild cython extensions if python, distutils, or cython change
|
|
||||||
cython_dependencies = [Value(v) for v in (sys.version, distutils.__version__, Cython.__version__)]
|
|
||||||
Export('cython_dependencies')
|
|
||||||
|
|
||||||
cereal_dir = Dir('.')
|
cereal_dir = Dir('.')
|
||||||
|
|
||||||
|
python_path = sysconfig.get_paths()['include']
|
||||||
cpppath = [
|
cpppath = [
|
||||||
'#',
|
'#',
|
||||||
'#cereal',
|
'#cereal',
|
||||||
"#cereal/messaging",
|
"#cereal/messaging",
|
||||||
"#opendbc/can",
|
"#opendbc/can",
|
||||||
'/usr/lib/include',
|
'/usr/lib/include',
|
||||||
|
python_path
|
||||||
]
|
]
|
||||||
|
|
||||||
AddOption('--test',
|
AddOption('--test',
|
||||||
|
@ -45,10 +41,14 @@ env = Environment(
|
||||||
] + ccflags_asan,
|
] + ccflags_asan,
|
||||||
LDFLAGS=ldflags_asan,
|
LDFLAGS=ldflags_asan,
|
||||||
LINKFLAGS=ldflags_asan,
|
LINKFLAGS=ldflags_asan,
|
||||||
|
LIBPATH=[
|
||||||
|
"#opendbc/can/",
|
||||||
|
],
|
||||||
CFLAGS="-std=gnu11",
|
CFLAGS="-std=gnu11",
|
||||||
CXXFLAGS="-std=c++1z",
|
CXXFLAGS="-std=c++1z",
|
||||||
CPPPATH=cpppath,
|
CPPPATH=cpppath,
|
||||||
|
CYTHONCFILESUFFIX=".cpp",
|
||||||
|
tools=["default", "cython"]
|
||||||
)
|
)
|
||||||
|
|
||||||
Export('env', 'zmq', 'arch')
|
Export('env', 'zmq', 'arch')
|
||||||
|
@ -57,5 +57,24 @@ cereal = [File('#cereal/libcereal.a')]
|
||||||
messaging = [File('#cereal/libmessaging.a')]
|
messaging = [File('#cereal/libmessaging.a')]
|
||||||
Export('cereal', 'messaging')
|
Export('cereal', 'messaging')
|
||||||
|
|
||||||
|
|
||||||
|
envCython = env.Clone()
|
||||||
|
envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-deprecated-declarations"]
|
||||||
|
|
||||||
|
python_libs = []
|
||||||
|
if arch == "Darwin":
|
||||||
|
envCython["LINKFLAGS"] = ["-bundle", "-undefined", "dynamic_lookup"]
|
||||||
|
elif arch == "aarch64":
|
||||||
|
envCython["LINKFLAGS"] = ["-shared"]
|
||||||
|
|
||||||
|
python_libs.append(os.path.basename(python_path))
|
||||||
|
else:
|
||||||
|
envCython["LINKFLAGS"] = ["-pthread", "-shared"]
|
||||||
|
|
||||||
|
envCython["LIBS"] = python_libs
|
||||||
|
|
||||||
|
Export('envCython')
|
||||||
|
|
||||||
|
|
||||||
SConscript(['cereal/SConscript'])
|
SConscript(['cereal/SConscript'])
|
||||||
SConscript(['opendbc/can/SConscript'])
|
SConscript(['opendbc/can/SConscript'])
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
Import('env', 'cereal', 'cython_dependencies')
|
Import('env', 'envCython', 'cereal')
|
||||||
|
|
||||||
import os
|
import os
|
||||||
from opendbc.can.process_dbc import process
|
from opendbc.can.process_dbc import process
|
||||||
|
@ -13,10 +13,10 @@ for x in sorted(os.listdir('../')):
|
||||||
dbc = env.Command(out_fn, in_fn, compile_dbc)
|
dbc = env.Command(out_fn, in_fn, compile_dbc)
|
||||||
dbcs.append(dbc)
|
dbcs.append(dbc)
|
||||||
|
|
||||||
|
|
||||||
libdbc = env.SharedLibrary('libdbc', ["dbc.cc", "parser.cc", "packer.cc", "common.cc"]+dbcs, LIBS=["capnp", "kj"])
|
libdbc = env.SharedLibrary('libdbc', ["dbc.cc", "parser.cc", "packer.cc", "common.cc"]+dbcs, LIBS=["capnp", "kj"])
|
||||||
|
|
||||||
# Build packer and parser
|
# Build packer and parser
|
||||||
env.Command(['packer_pyx.so', 'packer_pyx.cpp', 'parser_pyx.so', 'parser_pyx.cpp'],
|
lenv = envCython.Clone()
|
||||||
cython_dependencies + [libdbc, cereal, 'common_pyx_setup.py', 'common.pxd', 'packer_pyx.pyx', 'parser_pyx.pyx', 'packer.cc', 'parser.cc'],
|
lenv["LINKFLAGS"] += [libdbc[0].get_labspath()]
|
||||||
"cd opendbc/can && python3 common_pyx_setup.py build_ext --inplace")
|
lenv.Program('parser_pyx.so', 'parser_pyx.pyx')
|
||||||
|
lenv.Program('packer_pyx.so', 'packer_pyx.pyx')
|
||||||
|
|
|
@ -1,94 +0,0 @@
|
||||||
import os
|
|
||||||
import subprocess
|
|
||||||
import sysconfig
|
|
||||||
import platform
|
|
||||||
from distutils.core import Extension, setup # pylint: disable=import-error,no-name-in-module
|
|
||||||
|
|
||||||
from Cython.Build import cythonize
|
|
||||||
from Cython.Distutils import build_ext
|
|
||||||
|
|
||||||
|
|
||||||
ANNOTATE = os.getenv('ANNOTATE') is not None
|
|
||||||
BASEDIR = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../"))
|
|
||||||
|
|
||||||
|
|
||||||
def get_ext_filename_without_platform_suffix(filename):
|
|
||||||
name, ext = os.path.splitext(filename)
|
|
||||||
ext_suffix = sysconfig.get_config_var('EXT_SUFFIX')
|
|
||||||
|
|
||||||
if ext_suffix == ext:
|
|
||||||
return filename
|
|
||||||
|
|
||||||
ext_suffix = ext_suffix.replace(ext, '')
|
|
||||||
idx = name.find(ext_suffix)
|
|
||||||
|
|
||||||
if idx == -1:
|
|
||||||
return filename
|
|
||||||
else:
|
|
||||||
return name[:idx] + ext
|
|
||||||
|
|
||||||
|
|
||||||
class BuildExtWithoutPlatformSuffix(build_ext):
|
|
||||||
def get_ext_filename(self, ext_name):
|
|
||||||
filename = super().get_ext_filename(ext_name)
|
|
||||||
return get_ext_filename_without_platform_suffix(filename)
|
|
||||||
|
|
||||||
|
|
||||||
extra_compile_args = ["-std=c++1z", "-Wno-nullability-completeness"]
|
|
||||||
ARCH = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip() # pylint: disable=unexpected-keyword-arg
|
|
||||||
if ARCH == "aarch64":
|
|
||||||
extra_compile_args += ["-Wno-deprecated-register"]
|
|
||||||
|
|
||||||
if platform.system() == "Darwin":
|
|
||||||
libdbc = "libdbc.dylib"
|
|
||||||
else:
|
|
||||||
libdbc = "libdbc.so"
|
|
||||||
|
|
||||||
extra_link_args = [os.path.join(BASEDIR, 'opendbc', 'can', libdbc)]
|
|
||||||
include_dirs = [
|
|
||||||
BASEDIR,
|
|
||||||
os.path.join(BASEDIR, 'phonelibs'),
|
|
||||||
]
|
|
||||||
|
|
||||||
# Build CAN Parser
|
|
||||||
|
|
||||||
setup(name='CAN parser',
|
|
||||||
cmdclass={'build_ext': BuildExtWithoutPlatformSuffix},
|
|
||||||
ext_modules=cythonize(
|
|
||||||
Extension(
|
|
||||||
"parser_pyx",
|
|
||||||
language="c++",
|
|
||||||
sources=['parser_pyx.pyx'],
|
|
||||||
extra_compile_args=extra_compile_args,
|
|
||||||
include_dirs=include_dirs,
|
|
||||||
extra_link_args=extra_link_args,
|
|
||||||
),
|
|
||||||
nthreads=4,
|
|
||||||
annotate=ANNOTATE
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
if platform.system() == "Darwin":
|
|
||||||
os.system("install_name_tool -change opendbc/can/libdbc.dylib " + BASEDIR + "/opendbc/can/libdbc.dylib parser_pyx.so")
|
|
||||||
|
|
||||||
|
|
||||||
# Build CAN Packer
|
|
||||||
|
|
||||||
setup(name='CAN packer',
|
|
||||||
cmdclass={'build_ext': BuildExtWithoutPlatformSuffix},
|
|
||||||
ext_modules=cythonize(
|
|
||||||
Extension(
|
|
||||||
"packer_pyx",
|
|
||||||
language="c++",
|
|
||||||
sources=['packer_pyx.pyx'],
|
|
||||||
extra_compile_args=extra_compile_args,
|
|
||||||
include_dirs=include_dirs,
|
|
||||||
extra_link_args=extra_link_args,
|
|
||||||
),
|
|
||||||
nthreads=4,
|
|
||||||
annotate=ANNOTATE
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
if platform.system() == "Darwin":
|
|
||||||
os.system("install_name_tool -change opendbc/can/libdbc.dylib " + BASEDIR + "/opendbc/can/libdbc.dylib packer_pyx.so")
|
|
|
@ -8,8 +8,8 @@ from libcpp.string cimport string
|
||||||
from libcpp cimport bool
|
from libcpp cimport bool
|
||||||
from posix.dlfcn cimport dlopen, dlsym, RTLD_LAZY
|
from posix.dlfcn cimport dlopen, dlsym, RTLD_LAZY
|
||||||
|
|
||||||
from common cimport CANPacker as cpp_CANPacker
|
from .common cimport CANPacker as cpp_CANPacker
|
||||||
from common cimport dbc_lookup, SignalPackValue, DBC
|
from .common cimport dbc_lookup, SignalPackValue, DBC
|
||||||
|
|
||||||
|
|
||||||
cdef class CANPacker:
|
cdef class CANPacker:
|
||||||
|
@ -22,7 +22,7 @@ cdef class CANPacker:
|
||||||
def __init__(self, dbc_name):
|
def __init__(self, dbc_name):
|
||||||
self.dbc = dbc_lookup(dbc_name)
|
self.dbc = dbc_lookup(dbc_name)
|
||||||
if not self.dbc:
|
if not self.dbc:
|
||||||
raise RuntimeError("Can't lookup" + dbc_name)
|
raise RuntimeError(f"Can't lookup {dbc_name}")
|
||||||
|
|
||||||
self.packer = new cpp_CANPacker(dbc_name)
|
self.packer = new cpp_CANPacker(dbc_name)
|
||||||
num_msgs = self.dbc[0].num_msgs
|
num_msgs = self.dbc[0].num_msgs
|
||||||
|
|
|
@ -8,8 +8,8 @@ from libc.stdint cimport uint32_t, uint64_t, uint16_t
|
||||||
from libcpp.map cimport map
|
from libcpp.map cimport map
|
||||||
from libcpp cimport bool
|
from libcpp cimport bool
|
||||||
|
|
||||||
from common cimport CANParser as cpp_CANParser
|
from .common cimport CANParser as cpp_CANParser
|
||||||
from common cimport SignalParseOptions, MessageParseOptions, dbc_lookup, SignalValue, DBC
|
from .common cimport SignalParseOptions, MessageParseOptions, dbc_lookup, SignalValue, DBC
|
||||||
|
|
||||||
import os
|
import os
|
||||||
import numbers
|
import numbers
|
||||||
|
|
|
@ -0,0 +1,38 @@
|
||||||
|
import SCons
|
||||||
|
from SCons.Action import Action
|
||||||
|
|
||||||
|
cythonAction = Action("$CYTHONCOM")
|
||||||
|
|
||||||
|
def create_builder(env):
|
||||||
|
try:
|
||||||
|
cython = env['BUILDERS']['Cython']
|
||||||
|
except KeyError:
|
||||||
|
cython = SCons.Builder.Builder(
|
||||||
|
action = cythonAction,
|
||||||
|
emitter = {},
|
||||||
|
suffix = cython_suffix_emitter,
|
||||||
|
single_source = 1
|
||||||
|
)
|
||||||
|
env['BUILDERS']['Cython'] = cython
|
||||||
|
return cython
|
||||||
|
|
||||||
|
def cython_suffix_emitter(env, source):
|
||||||
|
return "$CYTHONCFILESUFFIX"
|
||||||
|
|
||||||
|
def generate(env):
|
||||||
|
env["CYTHON"] = "cythonize"
|
||||||
|
env["CYTHONCOM"] = "$CYTHON $CYTHONFLAGS $SOURCE"
|
||||||
|
env["CYTHONCFILESUFFIX"] = ".cpp"
|
||||||
|
|
||||||
|
c_file, _ = SCons.Tool.createCFileBuilders(env)
|
||||||
|
|
||||||
|
c_file.suffix['.pyx'] = cython_suffix_emitter
|
||||||
|
c_file.add_action('.pyx', cythonAction)
|
||||||
|
|
||||||
|
c_file.suffix['.py'] = cython_suffix_emitter
|
||||||
|
c_file.add_action('.py', cythonAction)
|
||||||
|
|
||||||
|
create_builder(env)
|
||||||
|
|
||||||
|
def exists(env):
|
||||||
|
return True
|
Loading…
Reference in New Issue