Merge pull request #4 from JoshuaArking/master
Rewrite of the J1979 Class with better PID support. Note from Brent: Great work and sorry for the slow response time. I was going to test that this doesn't break anything but I'm very busy right now. Merging and I'll let others raise the alarm if there are any problems.
This commit is contained in:
commit
09d951e93f
|
@ -89,6 +89,7 @@ venv/
|
|||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
.idea
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
|
|
|
@ -1,97 +1,25 @@
|
|||
from pandas import DataFrame, Series
|
||||
from numpy import int8, float16, uint8, uint16
|
||||
from numpy import dtype
|
||||
|
||||
|
||||
class J1979:
|
||||
def __init__(self, pid: int, original_data: DataFrame):
|
||||
def __init__(self, pid: int, original_data: DataFrame, pid_dict: DataFrame):
|
||||
self.pid: int = pid
|
||||
self.title: str = ""
|
||||
self.data: Series = self.process_response_data(original_data)
|
||||
self.title: str = pid_dict.at[pid, 'title']
|
||||
self.data: Series = self.process_response_data(original_data, pid_dict)
|
||||
print("Found " + str(self.data.shape[0]) + " responses for J1979 PID " + str(hex(self.pid)) + ":", self.title)
|
||||
|
||||
def process_response_data(self, original_data) -> Series:
|
||||
# ISO-TP formatted Universal Diagnostic Service (UDS) requests that were sent by the CAN collection device
|
||||
# during sampling. Request made using Arb ID 0x7DF with DLC of 8. Response should use Arb ID 7E8 (0x7DF + 0x8).
|
||||
|
||||
# DataFrame Columns: b0 b1 b2 b3 ... b7
|
||||
# -- -- -- -- --
|
||||
# PID 0x0C (12 dec) (Engine RPM): 02 01 0c 00 ... 00
|
||||
# PID 0x0D (13 dec) (Vehicle Speed): 02 01 0d 00 ... 00
|
||||
# PID 0x11 (17 dec) (Trottle Pos.): 02 01 11 00 ... 00
|
||||
# PID 0x61 (97 dec) (Demand % Torque): 02 01 61 00 ... 00
|
||||
# PID 0x62 (98 dec) (Engine % Torque): 02 01 62 00 ... 00
|
||||
# PID 0x63 (99 dec) (Ref. Torque): 02 01 63 00 ... 00
|
||||
# PID 0x8e (142 dec) (Friction Torque): 02 01 8e 00 ... 00
|
||||
|
||||
# Responses being managed here should follow the ISO-TP + UDS per-byte format AA BB CC DD .. DD
|
||||
# BYTE: AA BB CC DD ... DD
|
||||
# USE: response size (bytes) UDS mode + 0x40 UDS PID response data
|
||||
# DF COLUMN: b0 b1 b2 b3 ... b7
|
||||
|
||||
# Remember that this response data is already converted to decimal. Thus, byte BB = 65 = 0x41 = 0x01 + 0x40.
|
||||
# If BB isn't 0x41, check what the error code is. Some error code are listed in the UDS chapter of the car
|
||||
# hacker's handbook available at http://opengarages.org/handbook/ebook/.
|
||||
if self.pid == 12:
|
||||
self.title = 'Engine RPM'
|
||||
# PID is 0x0C: Engine RPM. 2 byte of data AA BB converted using 1/4 RPM per bit: (256*AA+BB)/4
|
||||
# Min value: 0 Max value: 16,383.75 units: rpm
|
||||
return Series(data=(256*original_data['b3']+original_data['b4'])/4,
|
||||
def process_response_data(self, original_data, pid_dict) -> Series:
|
||||
A = original_data['b3']
|
||||
B = original_data['b4']
|
||||
C = original_data['b5']
|
||||
D = original_data['b6']
|
||||
try:
|
||||
return Series(data=pid_dict.at[self.pid, 'formula'](A,B,C,D),
|
||||
index=original_data.index,
|
||||
name=self.title,
|
||||
dtype=float16)
|
||||
elif self.pid == 13:
|
||||
self.title = 'Speed km/h'
|
||||
# PID is 0x0D: Vehicle Speed. 1 byte of data AA using 1km/h per bit: no conversion necessary
|
||||
# Min value: 0 Max value: 255 (158.44965mph) units: km/h
|
||||
return Series(data=original_data['b3'],
|
||||
index=original_data.index,
|
||||
name=self.title,
|
||||
dtype=uint8)
|
||||
elif self.pid == 17:
|
||||
self.title = 'Throttle %'
|
||||
# PID is 0x11: Throttle Position. 1 byte of data AA using 100/255 % per bit: AA * 100/255% throttle.
|
||||
# Min value: 0 Max value: 100 units: %
|
||||
return Series(data=100 * original_data['b3'] / 255,
|
||||
index=original_data.index,
|
||||
name=self.title,
|
||||
dtype=uint8)
|
||||
elif self.pid == 97:
|
||||
self.title = 'Demand Torque %'
|
||||
# PID is 0x61: Driver's demand engine - percent torque. 1 byte of data AA using 1%/bit with -125 offset
|
||||
# AA - 125
|
||||
# Min value: -125 Max value: 130 units: %
|
||||
return Series(data=original_data['b3'] - 125,
|
||||
index=original_data.index,
|
||||
name=self.title,
|
||||
dtype=int8)
|
||||
elif self.pid == 98:
|
||||
self.title = 'Actual Torque %'
|
||||
# PID is 0x62: Actual engine - percent torque. 1 byte of data AA using 1%/bit with -125 offset
|
||||
# AA - 125
|
||||
# Min value: -125 Max value: 130 units: %
|
||||
return Series(data=original_data['b3'] - 125,
|
||||
index=original_data.index,
|
||||
name=self.title,
|
||||
dtype=int8)
|
||||
elif self.pid == 99:
|
||||
self.title = 'Reference Torque Nm'
|
||||
# PID is 0x63: Engine reference torque. 2 byte of data AA BB using 1 Nm/bit: 256*AA + BB Nm torque
|
||||
# Min value: 0 Max value: 65,535 units: Nm
|
||||
return Series(data=256*original_data['b3'] + original_data['b4'],
|
||||
index=original_data.index,
|
||||
name=self.title,
|
||||
dtype=uint16)
|
||||
elif self.pid == 142:
|
||||
self.title = 'Engine Friction Torque %'
|
||||
# PID is 0x8E: Engine Friction - Percent Torque. 1 byte of data AA using 1%/bit with -125 offset. AA - 125
|
||||
# Min value: -125 Max value: 130 units: %
|
||||
return Series(data=original_data['b3'] - 125,
|
||||
index=original_data.index,
|
||||
name=self.title,
|
||||
dtype=int8)
|
||||
else:
|
||||
# Looks like you were requesting J1979 data with your sniffer that hasn't been implemented in this code.
|
||||
# Time to do the leg work to expand this class accordingly then re-run the pipeline.
|
||||
dtype=dtype(pid_dict.at[self.pid, 'formula'](A,B,C,D)))
|
||||
except:
|
||||
raise ValueError("Encountered J1979 PID " + str(hex(self.pid)) +
|
||||
" in Pre-Processing that hasn't been programmed. Expand the J1979 class to handle all "
|
||||
" in Pre-Processing that hasn't been programmed. Expand the OBD2_pids.csv file to handle all "
|
||||
"J1979 requests made during data collection.")
|
||||
|
|
|
@ -12,7 +12,7 @@ for key, sample_list in samples.items(): # type: tuple, list
|
|||
for sample in sample_list: # type: Sample
|
||||
print(current_vehicle_number)
|
||||
print("\nData import and Pre-Processing for " + sample.output_vehicle_dir)
|
||||
id_dict, j1979_dict = sample.pre_process()
|
||||
id_dict, j1979_dict, pid_dict = sample.pre_process()
|
||||
if j1979_dict:
|
||||
sample.plot_j1979(j1979_dict, vehicle_number=str(current_vehicle_number))
|
||||
|
||||
|
|
|
@ -0,0 +1,56 @@
|
|||
4,Calculated engine load,A / 2.55
|
||||
5,Engine coolant temperature,A - 40
|
||||
6,Short term fuel trim—Bank 1,(A / 1.28) - 100
|
||||
7,Long term fuel trim—Bank 1,(A / 1.28) - 100
|
||||
8,Short term fuel trim—Bank 2,(A / 1.28) - 100
|
||||
9,Long term fuel trim—Bank 2,(A / 1.28) - 100
|
||||
10,Fuel pressure (gauge pressure),3 * A
|
||||
11,Intake manifold absolute pressure,A
|
||||
12,Engine RPM,((256 * A) + B)/4
|
||||
13,Vehicle speed,A
|
||||
14,Timing advance,(A / 2) - 64
|
||||
15,Intake air temperature,A - 40
|
||||
16,MAF air flow rate,((256 * A) + B)/100
|
||||
17,Throttle position,A / 2.55
|
||||
31,Run time since engine start,(256 * A) + B
|
||||
33,Distance traveled with malfunction indicator lamp (MIL) on,(256 * A) + B
|
||||
34,Fuel Rail Pressure (relative to manifold vacuum),0.079 * ((256 * A) + B)
|
||||
35,Fuel Rail Gauge Pressure (diesel, or gasoline direct injection),10 * ((256 * A) + B)
|
||||
44,Commanded EGR,A / 2.55
|
||||
45,EGR Error,(1.28 * A) - 100
|
||||
46,Commanded evaporative purge,A / 2.55
|
||||
47,Fuel Tank Level Input,A / 2.55
|
||||
48,Warm-ups since codes cleared,A
|
||||
49,Distance traveled since codes cleared,(256 * A) + B
|
||||
51,Absolute Barometric Pressure,A
|
||||
60,Catalyst Temperature: Bank 1, Sensor 1,(((256 * A) + B) / 10) - 40
|
||||
61,Catalyst Temperature: Bank 2, Sensor 1,(((256 * A) + B) / 10) - 40
|
||||
62,Catalyst Temperature: Bank 1, Sensor 2,(((256 * A) + B) / 10) - 40
|
||||
63,Catalyst Temperature: Bank 2, Sensor 2,(((256 * A) + B) / 10) - 40
|
||||
66,Control module voltage,((256 * A) + B) / 1000
|
||||
67,Absolute load value,((256 * A) + B) / 2.55
|
||||
68,Fuel–Air commanded equivalence ratio,(1 / 32,768) * ((256 * A) + B)
|
||||
69,Relative throttle position,A / 2.55
|
||||
70,Ambient air temperature,A - 40
|
||||
71,Absolute throttle position B,A / 2.55
|
||||
72,Absolute throttle position C,A / 2.55
|
||||
73,Accelerator pedal position D,A / 2.55
|
||||
74,Accelerator pedal position E,A / 2.55
|
||||
75,Accelerator pedal position F,A / 2.55
|
||||
76,Commanded throttle actuator,A / 2.55
|
||||
77,Time run with MIL on,(256 * A) + B
|
||||
78,Time since trouble codes cleared,(256 * A) + B
|
||||
80,Maximum value for air flow rate from mass air flow sensor,A * 10
|
||||
82,Ethanol fuel %,A / 2.55
|
||||
83,Absolute Evap system Vapor Pressure,((256 * A) + B) / 200
|
||||
84,Evap system vapor pressure,((256 * A) + B) - 32767
|
||||
89,Fuel rail absolute pressure,10 * ((256 * A) + B)
|
||||
90,Relative accelerator pedal position,A / 2.55
|
||||
91,Hybrid battery pack remaining life,A / 2.55
|
||||
92,Engine oil temperature,A - 40
|
||||
93,Fuel injection timing,(((256 * A) + B) / 128) - 210
|
||||
94,Engine fuel rate,((256 * A) + B) / 20
|
||||
97,Driver's demand engine - percent torque,A - 125
|
||||
98,Actual engine - percent torque,A - 125
|
||||
99,Engine reference torque,(256 * A) + B
|
||||
142,Engine Friction Torque %,A - 125
|
Can't render this file because it has a wrong number of fields in line 18.
|
|
@ -55,18 +55,46 @@ class PreProcessor:
|
|||
# print("\nSample of the original data:")
|
||||
# print(self.data.head(5), "\n")
|
||||
|
||||
def import_pid_dict(self, filename):
|
||||
# print("\nSample of the original data:")
|
||||
|
||||
# print(self.data.head(5), "\n")
|
||||
def pid(x):
|
||||
return int(x)
|
||||
|
||||
def title(x):
|
||||
return x
|
||||
|
||||
def formula(fx):
|
||||
f = lambda A, B, C, D: eval(fx)
|
||||
return f
|
||||
|
||||
# Used by pd.read_csv to apply the functions to the respective column vectors in the .csv file
|
||||
convert_dict = {'pid': pid, 'title': title, 'formula': formula}
|
||||
|
||||
print("\nReading in " + self.data_filename + "...")
|
||||
|
||||
return read_csv(filename,
|
||||
header=None,
|
||||
names=['pid', 'title', 'formula'],
|
||||
skiprows=0,
|
||||
delimiter=',',
|
||||
converters=convert_dict,
|
||||
index_col=0)
|
||||
|
||||
@staticmethod
|
||||
def generate_j1979_dictionary(j1979_data: DataFrame) -> dict:
|
||||
def generate_j1979_dictionary(j1979_data: DataFrame, pid_dict: DataFrame) -> dict:
|
||||
|
||||
d = {}
|
||||
services = j1979_data.groupby('b2')
|
||||
for uds_pid, data in services:
|
||||
d[uds_pid] = J1979(uds_pid, data)
|
||||
d[uds_pid] = J1979(uds_pid, data, pid_dict)
|
||||
return d
|
||||
|
||||
def generate_arb_id_dictionary(self,
|
||||
a_timer: PipelineTimer,
|
||||
normalize_strategy: Callable,
|
||||
pid_dict: DataFrame,
|
||||
time_conversion: int = 1000,
|
||||
freq_analysis_accuracy: float = 0.0,
|
||||
freq_synchronous_threshold: float = 0.0,
|
||||
|
@ -106,7 +134,7 @@ class PreProcessor:
|
|||
j1979_data.drop('dlc', axis=1, inplace=True)
|
||||
j1979_data.drop('id', axis=1, inplace=True)
|
||||
a_timer.start_nested_function_time()
|
||||
j1979_dictionary = self.generate_j1979_dictionary(j1979_data)
|
||||
j1979_dictionary = self.generate_j1979_dictionary(j1979_data, pid_dict)
|
||||
a_timer.set_j1979_creation()
|
||||
elif arb_id > 0:
|
||||
a_timer.start_iteration_time()
|
||||
|
@ -135,6 +163,10 @@ class PreProcessor:
|
|||
correction_mask = this_id.original_data.index.duplicated()
|
||||
this_id.original_data = this_id.original_data[~correction_mask]
|
||||
|
||||
# Check for non-monotonic values and sort them to be monotonic
|
||||
if not this_id.original_data.index.is_monotonic:
|
||||
this_id.original_data.sort_index(inplace=True)
|
||||
|
||||
this_id.generate_binary_matrix_and_tang(a_timer, normalize_strategy)
|
||||
this_id.analyze_transmission_frequency(time_convert=time_conversion,
|
||||
ci_accuracy=freq_analysis_accuracy,
|
||||
|
|
|
@ -115,8 +115,14 @@ class Sample:
|
|||
def pre_process(self):
|
||||
self.make_and_move_to_vehicle_directory()
|
||||
pre_processor = PreProcessor(self.path, pickle_arb_id_filename, pickle_j1979_filename, self.use_j1979)
|
||||
|
||||
self.move_back_to_parent_directory()
|
||||
pid_dictionary = pre_processor.import_pid_dict('OBD2_pids.csv')
|
||||
self.make_and_move_to_vehicle_directory()
|
||||
|
||||
id_dictionary, j1979_dictionary = pre_processor.generate_arb_id_dictionary(a_timer,
|
||||
tang_normalize_strategy,
|
||||
pid_dictionary,
|
||||
time_conversion,
|
||||
freq_analysis_accuracy,
|
||||
freq_synchronous_threshold,
|
||||
|
@ -140,7 +146,7 @@ class Sample:
|
|||
dump(j1979_dictionary, open(pickle_j1979_filename, "wb"))
|
||||
print("\tComplete...")
|
||||
self.move_back_to_parent_directory()
|
||||
return id_dictionary, j1979_dictionary
|
||||
return id_dictionary, j1979_dictionary, pid_dictionary
|
||||
|
||||
def plot_j1979(self, j1979_dictionary: dict, vehicle_number: str):
|
||||
self.make_and_move_to_vehicle_directory()
|
||||
|
|
Loading…
Reference in New Issue