diff --git a/.gitignore b/.gitignore index 894a44c..3a99b0f 100644 --- a/.gitignore +++ b/.gitignore @@ -89,6 +89,7 @@ venv/ ENV/ env.bak/ venv.bak/ +.idea # Spyder project settings .spyderproject diff --git a/Pipeline_multi-file/J1979.py b/Pipeline_multi-file/J1979.py index 6c1730c..a0364ca 100644 --- a/Pipeline_multi-file/J1979.py +++ b/Pipeline_multi-file/J1979.py @@ -1,97 +1,25 @@ from pandas import DataFrame, Series -from numpy import int8, float16, uint8, uint16 +from numpy import dtype class J1979: - def __init__(self, pid: int, original_data: DataFrame): + def __init__(self, pid: int, original_data: DataFrame, pid_dict: DataFrame): self.pid: int = pid - self.title: str = "" - self.data: Series = self.process_response_data(original_data) + self.title: str = pid_dict.at[pid, 'title'] + self.data: Series = self.process_response_data(original_data, pid_dict) print("Found " + str(self.data.shape[0]) + " responses for J1979 PID " + str(hex(self.pid)) + ":", self.title) - def process_response_data(self, original_data) -> Series: - # ISO-TP formatted Universal Diagnostic Service (UDS) requests that were sent by the CAN collection device - # during sampling. Request made using Arb ID 0x7DF with DLC of 8. Response should use Arb ID 7E8 (0x7DF + 0x8). - - # DataFrame Columns: b0 b1 b2 b3 ... b7 - # -- -- -- -- -- - # PID 0x0C (12 dec) (Engine RPM): 02 01 0c 00 ... 00 - # PID 0x0D (13 dec) (Vehicle Speed): 02 01 0d 00 ... 00 - # PID 0x11 (17 dec) (Trottle Pos.): 02 01 11 00 ... 00 - # PID 0x61 (97 dec) (Demand % Torque): 02 01 61 00 ... 00 - # PID 0x62 (98 dec) (Engine % Torque): 02 01 62 00 ... 00 - # PID 0x63 (99 dec) (Ref. Torque): 02 01 63 00 ... 00 - # PID 0x8e (142 dec) (Friction Torque): 02 01 8e 00 ... 00 - - # Responses being managed here should follow the ISO-TP + UDS per-byte format AA BB CC DD .. DD - # BYTE: AA BB CC DD ... DD - # USE: response size (bytes) UDS mode + 0x40 UDS PID response data - # DF COLUMN: b0 b1 b2 b3 ... b7 - - # Remember that this response data is already converted to decimal. Thus, byte BB = 65 = 0x41 = 0x01 + 0x40. - # If BB isn't 0x41, check what the error code is. Some error code are listed in the UDS chapter of the car - # hacker's handbook available at http://opengarages.org/handbook/ebook/. - if self.pid == 12: - self.title = 'Engine RPM' - # PID is 0x0C: Engine RPM. 2 byte of data AA BB converted using 1/4 RPM per bit: (256*AA+BB)/4 - # Min value: 0 Max value: 16,383.75 units: rpm - return Series(data=(256*original_data['b3']+original_data['b4'])/4, + def process_response_data(self, original_data, pid_dict) -> Series: + A = original_data['b3'] + B = original_data['b4'] + C = original_data['b5'] + D = original_data['b6'] + try: + return Series(data=pid_dict.at[self.pid, 'formula'](A,B,C,D), index=original_data.index, name=self.title, - dtype=float16) - elif self.pid == 13: - self.title = 'Speed km/h' - # PID is 0x0D: Vehicle Speed. 1 byte of data AA using 1km/h per bit: no conversion necessary - # Min value: 0 Max value: 255 (158.44965mph) units: km/h - return Series(data=original_data['b3'], - index=original_data.index, - name=self.title, - dtype=uint8) - elif self.pid == 17: - self.title = 'Throttle %' - # PID is 0x11: Throttle Position. 1 byte of data AA using 100/255 % per bit: AA * 100/255% throttle. - # Min value: 0 Max value: 100 units: % - return Series(data=100 * original_data['b3'] / 255, - index=original_data.index, - name=self.title, - dtype=uint8) - elif self.pid == 97: - self.title = 'Demand Torque %' - # PID is 0x61: Driver's demand engine - percent torque. 1 byte of data AA using 1%/bit with -125 offset - # AA - 125 - # Min value: -125 Max value: 130 units: % - return Series(data=original_data['b3'] - 125, - index=original_data.index, - name=self.title, - dtype=int8) - elif self.pid == 98: - self.title = 'Actual Torque %' - # PID is 0x62: Actual engine - percent torque. 1 byte of data AA using 1%/bit with -125 offset - # AA - 125 - # Min value: -125 Max value: 130 units: % - return Series(data=original_data['b3'] - 125, - index=original_data.index, - name=self.title, - dtype=int8) - elif self.pid == 99: - self.title = 'Reference Torque Nm' - # PID is 0x63: Engine reference torque. 2 byte of data AA BB using 1 Nm/bit: 256*AA + BB Nm torque - # Min value: 0 Max value: 65,535 units: Nm - return Series(data=256*original_data['b3'] + original_data['b4'], - index=original_data.index, - name=self.title, - dtype=uint16) - elif self.pid == 142: - self.title = 'Engine Friction Torque %' - # PID is 0x8E: Engine Friction - Percent Torque. 1 byte of data AA using 1%/bit with -125 offset. AA - 125 - # Min value: -125 Max value: 130 units: % - return Series(data=original_data['b3'] - 125, - index=original_data.index, - name=self.title, - dtype=int8) - else: - # Looks like you were requesting J1979 data with your sniffer that hasn't been implemented in this code. - # Time to do the leg work to expand this class accordingly then re-run the pipeline. + dtype=dtype(pid_dict.at[self.pid, 'formula'](A,B,C,D))) + except: raise ValueError("Encountered J1979 PID " + str(hex(self.pid)) + - " in Pre-Processing that hasn't been programmed. Expand the J1979 class to handle all " + " in Pre-Processing that hasn't been programmed. Expand the OBD2_pids.csv file to handle all " "J1979 requests made during data collection.") diff --git a/Pipeline_multi-file/Main.py b/Pipeline_multi-file/Main.py index 0f1d030..81a3355 100644 --- a/Pipeline_multi-file/Main.py +++ b/Pipeline_multi-file/Main.py @@ -12,7 +12,7 @@ for key, sample_list in samples.items(): # type: tuple, list for sample in sample_list: # type: Sample print(current_vehicle_number) print("\nData import and Pre-Processing for " + sample.output_vehicle_dir) - id_dict, j1979_dict = sample.pre_process() + id_dict, j1979_dict, pid_dict = sample.pre_process() if j1979_dict: sample.plot_j1979(j1979_dict, vehicle_number=str(current_vehicle_number)) diff --git a/Pipeline_multi-file/OBD2_pids.csv b/Pipeline_multi-file/OBD2_pids.csv new file mode 100644 index 0000000..67f3e23 --- /dev/null +++ b/Pipeline_multi-file/OBD2_pids.csv @@ -0,0 +1,56 @@ +4,Calculated engine load,A / 2.55 +5,Engine coolant temperature,A - 40 +6,Short term fuel trim—Bank 1,(A / 1.28) - 100 +7,Long term fuel trim—Bank 1,(A / 1.28) - 100 +8,Short term fuel trim—Bank 2,(A / 1.28) - 100 +9,Long term fuel trim—Bank 2,(A / 1.28) - 100 +10,Fuel pressure (gauge pressure),3 * A +11,Intake manifold absolute pressure,A +12,Engine RPM,((256 * A) + B)/4 +13,Vehicle speed,A +14,Timing advance,(A / 2) - 64 +15,Intake air temperature,A - 40 +16,MAF air flow rate,((256 * A) + B)/100 +17,Throttle position,A / 2.55 +31,Run time since engine start,(256 * A) + B +33,Distance traveled with malfunction indicator lamp (MIL) on,(256 * A) + B +34,Fuel Rail Pressure (relative to manifold vacuum),0.079 * ((256 * A) + B) +35,Fuel Rail Gauge Pressure (diesel, or gasoline direct injection),10 * ((256 * A) + B) +44,Commanded EGR,A / 2.55 +45,EGR Error,(1.28 * A) - 100 +46,Commanded evaporative purge,A / 2.55 +47,Fuel Tank Level Input,A / 2.55 +48,Warm-ups since codes cleared,A +49,Distance traveled since codes cleared,(256 * A) + B +51,Absolute Barometric Pressure,A +60,Catalyst Temperature: Bank 1, Sensor 1,(((256 * A) + B) / 10) - 40 +61,Catalyst Temperature: Bank 2, Sensor 1,(((256 * A) + B) / 10) - 40 +62,Catalyst Temperature: Bank 1, Sensor 2,(((256 * A) + B) / 10) - 40 +63,Catalyst Temperature: Bank 2, Sensor 2,(((256 * A) + B) / 10) - 40 +66,Control module voltage,((256 * A) + B) / 1000 +67,Absolute load value,((256 * A) + B) / 2.55 +68,Fuel–Air commanded equivalence ratio,(1 / 32,768) * ((256 * A) + B) +69,Relative throttle position,A / 2.55 +70,Ambient air temperature,A - 40 +71,Absolute throttle position B,A / 2.55 +72,Absolute throttle position C,A / 2.55 +73,Accelerator pedal position D,A / 2.55 +74,Accelerator pedal position E,A / 2.55 +75,Accelerator pedal position F,A / 2.55 +76,Commanded throttle actuator,A / 2.55 +77,Time run with MIL on,(256 * A) + B +78,Time since trouble codes cleared,(256 * A) + B +80,Maximum value for air flow rate from mass air flow sensor,A * 10 +82,Ethanol fuel %,A / 2.55 +83,Absolute Evap system Vapor Pressure,((256 * A) + B) / 200 +84,Evap system vapor pressure,((256 * A) + B) - 32767 +89,Fuel rail absolute pressure,10 * ((256 * A) + B) +90,Relative accelerator pedal position,A / 2.55 +91,Hybrid battery pack remaining life,A / 2.55 +92,Engine oil temperature,A - 40 +93,Fuel injection timing,(((256 * A) + B) / 128) - 210 +94,Engine fuel rate,((256 * A) + B) / 20 +97,Driver's demand engine - percent torque,A - 125 +98,Actual engine - percent torque,A - 125 +99,Engine reference torque,(256 * A) + B +142,Engine Friction Torque %,A - 125 diff --git a/Pipeline_multi-file/PreProcessor.py b/Pipeline_multi-file/PreProcessor.py index de1ee20..100908d 100644 --- a/Pipeline_multi-file/PreProcessor.py +++ b/Pipeline_multi-file/PreProcessor.py @@ -55,18 +55,46 @@ class PreProcessor: # print("\nSample of the original data:") # print(self.data.head(5), "\n") + def import_pid_dict(self, filename): + # print("\nSample of the original data:") + + # print(self.data.head(5), "\n") + def pid(x): + return int(x) + + def title(x): + return x + + def formula(fx): + f = lambda A, B, C, D: eval(fx) + return f + + # Used by pd.read_csv to apply the functions to the respective column vectors in the .csv file + convert_dict = {'pid': pid, 'title': title, 'formula': formula} + + print("\nReading in " + self.data_filename + "...") + + return read_csv(filename, + header=None, + names=['pid', 'title', 'formula'], + skiprows=0, + delimiter=',', + converters=convert_dict, + index_col=0) + @staticmethod - def generate_j1979_dictionary(j1979_data: DataFrame) -> dict: + def generate_j1979_dictionary(j1979_data: DataFrame, pid_dict: DataFrame) -> dict: d = {} services = j1979_data.groupby('b2') for uds_pid, data in services: - d[uds_pid] = J1979(uds_pid, data) + d[uds_pid] = J1979(uds_pid, data, pid_dict) return d def generate_arb_id_dictionary(self, a_timer: PipelineTimer, normalize_strategy: Callable, + pid_dict: DataFrame, time_conversion: int = 1000, freq_analysis_accuracy: float = 0.0, freq_synchronous_threshold: float = 0.0, @@ -106,7 +134,7 @@ class PreProcessor: j1979_data.drop('dlc', axis=1, inplace=True) j1979_data.drop('id', axis=1, inplace=True) a_timer.start_nested_function_time() - j1979_dictionary = self.generate_j1979_dictionary(j1979_data) + j1979_dictionary = self.generate_j1979_dictionary(j1979_data, pid_dict) a_timer.set_j1979_creation() elif arb_id > 0: a_timer.start_iteration_time() @@ -135,6 +163,10 @@ class PreProcessor: correction_mask = this_id.original_data.index.duplicated() this_id.original_data = this_id.original_data[~correction_mask] + # Check for non-monotonic values and sort them to be monotonic + if not this_id.original_data.index.is_monotonic: + this_id.original_data.sort_index(inplace=True) + this_id.generate_binary_matrix_and_tang(a_timer, normalize_strategy) this_id.analyze_transmission_frequency(time_convert=time_conversion, ci_accuracy=freq_analysis_accuracy, diff --git a/Pipeline_multi-file/Sample.py b/Pipeline_multi-file/Sample.py index 488bf1e..9c86229 100644 --- a/Pipeline_multi-file/Sample.py +++ b/Pipeline_multi-file/Sample.py @@ -115,8 +115,14 @@ class Sample: def pre_process(self): self.make_and_move_to_vehicle_directory() pre_processor = PreProcessor(self.path, pickle_arb_id_filename, pickle_j1979_filename, self.use_j1979) + + self.move_back_to_parent_directory() + pid_dictionary = pre_processor.import_pid_dict('OBD2_pids.csv') + self.make_and_move_to_vehicle_directory() + id_dictionary, j1979_dictionary = pre_processor.generate_arb_id_dictionary(a_timer, tang_normalize_strategy, + pid_dictionary, time_conversion, freq_analysis_accuracy, freq_synchronous_threshold, @@ -140,7 +146,7 @@ class Sample: dump(j1979_dictionary, open(pickle_j1979_filename, "wb")) print("\tComplete...") self.move_back_to_parent_directory() - return id_dictionary, j1979_dictionary + return id_dictionary, j1979_dictionary, pid_dictionary def plot_j1979(self, j1979_dictionary: dict, vehicle_number: str): self.make_and_move_to_vehicle_directory()