rusefi
/
CAN_Reverse_Engineering
mirror of https://github.com/rusefi/CAN_Reverse_Engineering.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #4 from JoshuaArking/master 

Rewrite of the J1979 Class with better PID support.

Note from Brent: Great work and sorry for the slow response time. I was going to test that this doesn't break anything but I'm very busy right now. Merging and I'll let others raise the alarm if there are any problems.
This commit is contained in:
brent-stone 2019-10-23 23:26:50 -05:00 committed by GitHub
parent 99f55ced91 315eeabc13
commit 09d951e93f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 114 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -89,6 +89,7 @@ venv/
ENV/
env.bak/
venv.bak/
.idea
# Spyder project settings
.spyderproject

100
Pipeline_multi-file/J1979.py
View File

@ -1,97 +1,25 @@
from pandas import DataFrame, Series
from numpy import int8, float16, uint8, uint16
from numpy import dtype
class J1979:
def __init__(self, pid: int, original_data: DataFrame):
def __init__(self, pid: int, original_data: DataFrame, pid_dict: DataFrame):
self.pid: int = pid
self.title: str = ""
self.data: Series = self.process_response_data(original_data)
self.title: str = pid_dict.at[pid, 'title']
self.data: Series = self.process_response_data(original_data, pid_dict)
print("Found " + str(self.data.shape[0]) + " responses for J1979 PID " + str(hex(self.pid)) + ":", self.title)
def process_response_data(self, original_data) -> Series:
# ISO-TP formatted Universal Diagnostic Service (UDS) requests that were sent by the CAN collection device
# during sampling. Request made using Arb ID 0x7DF with DLC of 8. Response should use Arb ID 7E8 (0x7DF + 0x8).
# DataFrame Columns: b0 b1 b2 b3 ... b7
# -- -- -- -- --
# PID 0x0C (12 dec) (Engine RPM): 02 01 0c 00 ... 00
# PID 0x0D (13 dec) (Vehicle Speed): 02 01 0d 00 ... 00
# PID 0x11 (17 dec) (Trottle Pos.): 02 01 11 00 ... 00
# PID 0x61 (97 dec) (Demand % Torque): 02 01 61 00 ... 00
# PID 0x62 (98 dec) (Engine % Torque): 02 01 62 00 ... 00
# PID 0x63 (99 dec) (Ref. Torque): 02 01 63 00 ... 00
# PID 0x8e (142 dec) (Friction Torque): 02 01 8e 00 ... 00
# Responses being managed here should follow the ISO-TP + UDS per-byte format AA BB CC DD .. DD
# BYTE: AA BB CC DD ... DD
# USE: response size (bytes) UDS mode + 0x40 UDS PID response data
# DF COLUMN: b0 b1 b2 b3 ... b7
# Remember that this response data is already converted to decimal. Thus, byte BB = 65 = 0x41 = 0x01 + 0x40.
# If BB isn't 0x41, check what the error code is. Some error code are listed in the UDS chapter of the car
# hacker's handbook available at http://opengarages.org/handbook/ebook/.
if self.pid == 12:
self.title = 'Engine RPM'
# PID is 0x0C: Engine RPM. 2 byte of data AA BB converted using 1/4 RPM per bit: (256*AA+BB)/4
# Min value: 0 Max value: 16,383.75 units: rpm
return Series(data=(256*original_data['b3']+original_data['b4'])/4,
def process_response_data(self, original_data, pid_dict) -> Series:
A = original_data['b3']
B = original_data['b4']
C = original_data['b5']
D = original_data['b6']
try:
return Series(data=pid_dict.at[self.pid, 'formula'](A,B,C,D),
index=original_data.index,
name=self.title,
dtype=float16)
elif self.pid == 13:
self.title = 'Speed km/h'
# PID is 0x0D: Vehicle Speed. 1 byte of data AA using 1km/h per bit: no conversion necessary
# Min value: 0 Max value: 255 (158.44965mph) units: km/h
return Series(data=original_data['b3'],
index=original_data.index,
name=self.title,
dtype=uint8)
elif self.pid == 17:
self.title = 'Throttle %'
# PID is 0x11: Throttle Position. 1 byte of data AA using 100/255 % per bit: AA * 100/255% throttle.
# Min value: 0 Max value: 100 units: %
return Series(data=100 * original_data['b3'] / 255,
index=original_data.index,
name=self.title,
dtype=uint8)
elif self.pid == 97:
self.title = 'Demand Torque %'
# PID is 0x61: Driver's demand engine - percent torque. 1 byte of data AA using 1%/bit with -125 offset
# AA - 125
# Min value: -125 Max value: 130 units: %
return Series(data=original_data['b3'] - 125,
index=original_data.index,
name=self.title,
dtype=int8)
elif self.pid == 98:
self.title = 'Actual Torque %'
# PID is 0x62: Actual engine - percent torque. 1 byte of data AA using 1%/bit with -125 offset
# AA - 125
# Min value: -125 Max value: 130 units: %
return Series(data=original_data['b3'] - 125,
index=original_data.index,
name=self.title,
dtype=int8)
elif self.pid == 99:
self.title = 'Reference Torque Nm'
# PID is 0x63: Engine reference torque. 2 byte of data AA BB using 1 Nm/bit: 256*AA + BB Nm torque
# Min value: 0 Max value: 65,535 units: Nm
return Series(data=256*original_data['b3'] + original_data['b4'],
index=original_data.index,
name=self.title,
dtype=uint16)
elif self.pid == 142:
self.title = 'Engine Friction Torque %'
# PID is 0x8E: Engine Friction - Percent Torque. 1 byte of data AA using 1%/bit with -125 offset. AA - 125
# Min value: -125 Max value: 130 units: %
return Series(data=original_data['b3'] - 125,
index=original_data.index,
name=self.title,
dtype=int8)
else:
# Looks like you were requesting J1979 data with your sniffer that hasn't been implemented in this code.
# Time to do the leg work to expand this class accordingly then re-run the pipeline.
dtype=dtype(pid_dict.at[self.pid, 'formula'](A,B,C,D)))
except:
raise ValueError("Encountered J1979 PID " + str(hex(self.pid)) +
" in Pre-Processing that hasn't been programmed. Expand the J1979 class to handle all "
" in Pre-Processing that hasn't been programmed. Expand the OBD2_pids.csv file to handle all "
"J1979 requests made during data collection.")

2
Pipeline_multi-file/Main.py
View File

@ -12,7 +12,7 @@ for key, sample_list in samples.items(): # type: tuple, list
for sample in sample_list: # type: Sample
print(current_vehicle_number)
print("\nData import and Pre-Processing for " + sample.output_vehicle_dir)
id_dict, j1979_dict = sample.pre_process()
id_dict, j1979_dict, pid_dict = sample.pre_process()
if j1979_dict:
sample.plot_j1979(j1979_dict, vehicle_number=str(current_vehicle_number))

56
Pipeline_multi-file/OBD2_pids.csv Normal file
View File

@ -0,0 +1,56 @@
4,Calculated engine load,A / 2.55
5,Engine coolant temperature,A - 40
6,Short term fuel trim—Bank 1,(A / 1.28) - 100
7,Long term fuel trim—Bank 1,(A / 1.28) - 100
8,Short term fuel trim—Bank 2,(A / 1.28) - 100
9,Long term fuel trim—Bank 2,(A / 1.28) - 100
10,Fuel pressure (gauge pressure),3 * A
11,Intake manifold absolute pressure,A
12,Engine RPM,((256 * A) + B)/4
13,Vehicle speed,A
14,Timing advance,(A / 2) - 64
15,Intake air temperature,A - 40
16,MAF air flow rate,((256 * A) + B)/100
17,Throttle position,A / 2.55
31,Run time since engine start,(256 * A) + B
33,Distance traveled with malfunction indicator lamp (MIL) on,(256 * A) + B
34,Fuel Rail Pressure (relative to manifold vacuum),0.079 * ((256 * A) + B)
35,Fuel Rail Gauge Pressure (diesel, or gasoline direct injection),10 * ((256 * A) + B)
44,Commanded EGR,A / 2.55
45,EGR Error,(1.28 * A) - 100
46,Commanded evaporative purge,A / 2.55
47,Fuel Tank Level Input,A / 2.55
48,Warm-ups since codes cleared,A
49,Distance traveled since codes cleared,(256 * A) + B
51,Absolute Barometric Pressure,A
60,Catalyst Temperature: Bank 1, Sensor 1,(((256 * A) + B) / 10) - 40
61,Catalyst Temperature: Bank 2, Sensor 1,(((256 * A) + B) / 10) - 40
62,Catalyst Temperature: Bank 1, Sensor 2,(((256 * A) + B) / 10) - 40
63,Catalyst Temperature: Bank 2, Sensor 2,(((256 * A) + B) / 10) - 40
66,Control module voltage,((256 * A) + B) / 1000
67,Absolute load value,((256 * A) + B) / 2.55
68,Fuel–Air commanded equivalence ratio,(1 / 32,768) * ((256 * A) + B)
69,Relative throttle position,A / 2.55
70,Ambient air temperature,A - 40
71,Absolute throttle position B,A / 2.55
72,Absolute throttle position C,A / 2.55
73,Accelerator pedal position D,A / 2.55
74,Accelerator pedal position E,A / 2.55
75,Accelerator pedal position F,A / 2.55
76,Commanded throttle actuator,A / 2.55
77,Time run with MIL on,(256 * A) + B
78,Time since trouble codes cleared,(256 * A) + B
80,Maximum value for air flow rate from mass air flow sensor,A * 10
82,Ethanol fuel %,A / 2.55
83,Absolute Evap system Vapor Pressure,((256 * A) + B) / 200
84,Evap system vapor pressure,((256 * A) + B) - 32767
89,Fuel rail absolute pressure,10 * ((256 * A) + B)
90,Relative accelerator pedal position,A / 2.55
91,Hybrid battery pack remaining life,A / 2.55
92,Engine oil temperature,A - 40
93,Fuel injection timing,(((256 * A) + B) / 128) - 210
94,Engine fuel rate,((256 * A) + B) / 20
97,Driver's demand engine - percent torque,A - 125
98,Actual engine - percent torque,A - 125
99,Engine reference torque,(256 * A) + B
142,Engine Friction Torque %,A - 125
Can't render this file because it has a wrong number of fields in line 18.

38
Pipeline_multi-file/PreProcessor.py
View File

@ -55,18 +55,46 @@ class PreProcessor:
# print("\nSample of the original data:")
# print(self.data.head(5), "\n")
def import_pid_dict(self, filename):
# print("\nSample of the original data:")
# print(self.data.head(5), "\n")
def pid(x):
return int(x)
def title(x):
return x
def formula(fx):
f = lambda A, B, C, D: eval(fx)
return f
# Used by pd.read_csv to apply the functions to the respective column vectors in the .csv file
convert_dict = {'pid': pid, 'title': title, 'formula': formula}
print("\nReading in " + self.data_filename + "...")
return read_csv(filename,
header=None,
names=['pid', 'title', 'formula'],
skiprows=0,
delimiter=',',
converters=convert_dict,
index_col=0)
@staticmethod
def generate_j1979_dictionary(j1979_data: DataFrame) -> dict:
def generate_j1979_dictionary(j1979_data: DataFrame, pid_dict: DataFrame) -> dict:
d = {}
services = j1979_data.groupby('b2')
for uds_pid, data in services:
d[uds_pid] = J1979(uds_pid, data)
d[uds_pid] = J1979(uds_pid, data, pid_dict)
return d
def generate_arb_id_dictionary(self,
a_timer: PipelineTimer,
normalize_strategy: Callable,
pid_dict: DataFrame,
time_conversion: int = 1000,
freq_analysis_accuracy: float = 0.0,
freq_synchronous_threshold: float = 0.0,
@ -106,7 +134,7 @@ class PreProcessor:
j1979_data.drop('dlc', axis=1, inplace=True)
j1979_data.drop('id', axis=1, inplace=True)
a_timer.start_nested_function_time()
j1979_dictionary = self.generate_j1979_dictionary(j1979_data)
j1979_dictionary = self.generate_j1979_dictionary(j1979_data, pid_dict)
a_timer.set_j1979_creation()
elif arb_id > 0:
a_timer.start_iteration_time()
@ -135,6 +163,10 @@ class PreProcessor:
correction_mask = this_id.original_data.index.duplicated()
this_id.original_data = this_id.original_data[~correction_mask]
# Check for non-monotonic values and sort them to be monotonic
if not this_id.original_data.index.is_monotonic:
this_id.original_data.sort_index(inplace=True)
this_id.generate_binary_matrix_and_tang(a_timer, normalize_strategy)
this_id.analyze_transmission_frequency(time_convert=time_conversion,
ci_accuracy=freq_analysis_accuracy,

8
Pipeline_multi-file/Sample.py
View File

@ -115,8 +115,14 @@ class Sample:
def pre_process(self):
self.make_and_move_to_vehicle_directory()
pre_processor = PreProcessor(self.path, pickle_arb_id_filename, pickle_j1979_filename, self.use_j1979)
self.move_back_to_parent_directory()
pid_dictionary = pre_processor.import_pid_dict('OBD2_pids.csv')
self.make_and_move_to_vehicle_directory()
id_dictionary, j1979_dictionary = pre_processor.generate_arb_id_dictionary(a_timer,
tang_normalize_strategy,
pid_dictionary,
time_conversion,
freq_analysis_accuracy,
freq_synchronous_threshold,
@ -140,7 +146,7 @@ class Sample:
dump(j1979_dictionary, open(pickle_j1979_filename, "wb"))
print("\tComplete...")
self.move_back_to_parent_directory()
return id_dictionary, j1979_dictionary
return id_dictionary, j1979_dictionary, pid_dictionary
def plot_j1979(self, j1979_dictionary: dict, vehicle_number: str):
self.make_and_move_to_vehicle_directory()