diff --git a/gr-tempest/examples/simulations/automatic_simulated_tempest_TMDS_example.grc b/gr-tempest/examples/simulations/automatic_simulated_tempest_TMDS_example.grc index c911792..46c68e3 100644 --- a/gr-tempest/examples/simulations/automatic_simulated_tempest_TMDS_example.grc +++ b/gr-tempest/examples/simulations/automatic_simulated_tempest_TMDS_example.grc @@ -98,7 +98,7 @@ blocks: start: '0' step: '1' stop: int(4096*1.5) - value: '1056' + value: '1800' widget: counter states: bus_sink: false @@ -111,7 +111,7 @@ blocks: id: variable parameters: comment: '' - value: '800' + value: '1600' states: bus_sink: false bus_source: false @@ -143,7 +143,7 @@ blocks: start: '0' step: '1' stop: int(2160*1.5) - value: '628' + value: '1000' widget: counter states: bus_sink: false @@ -156,7 +156,7 @@ blocks: id: variable parameters: comment: '' - value: '600' + value: '900' states: bus_sink: false bus_source: false @@ -788,7 +788,7 @@ blocks: alias: '' blanking: 'True' comment: '' - image_file: /home/emidan19/deep-tempest/images/VAMO!!.png + image_file: /home/emidan19/deep-tempest/images/1600x900_google.png maxoutbuf: '0' minoutbuf: '0' mode: '1' @@ -840,25 +840,6 @@ blocks: coordinate: [2048, 924.0] rotation: 180 state: enabled -- name: tempest_sync_detector_0_0 - id: tempest_sync_detector - parameters: - affinity: '' - alias: '' - comment: '' - hblanking: interpolatedHblank - hscreen: interpolatedHscreen - maxoutbuf: '0' - minoutbuf: '0' - vblanking: Vblank - vscreen: Vsize-Vblank - states: - bus_sink: false - bus_source: false - bus_structure: null - coordinate: [1440, 676.0] - rotation: 0 - state: enabled - name: uhd_usrp_source_0 id: uhd_usrp_source parameters: @@ -1256,7 +1237,7 @@ blocks: connections: - [FFT_autocorrelation_0_0, '0', tempest_fft_peak_fine_sampling_sync_0_0, '0'] - [Keep_1_in_N_Frames_0, '0', FFT_autocorrelation_0_0, '0'] -- [Keep_1_in_N_Frames_0, '0', tempest_sync_detector_0_0, '0'] +- [Keep_1_in_N_Frames_0, '0', blocks_delay_0_0, '0'] - [analog_sig_source_x_0, '0', blocks_multiply_xx_0, '1'] - [binary_serializer_0, '0', blocks_add_xx_0, '0'] - [binary_serializer_0, '0', blocks_float_to_complex_0, '0'] @@ -1279,10 +1260,8 @@ connections: - [tempest_TMDS_image_source_0, '2', blocks_null_sink_0, '1'] - [tempest_fft_peak_fine_sampling_sync_0_0, '0', qtgui_time_sink_x_1, '0'] - [tempest_fft_peak_fine_sampling_sync_0_0, en, FFT_autocorrelation_0_0, en] -- [tempest_fft_peak_fine_sampling_sync_0_0, en, tempest_sync_detector_0_0, en] - [tempest_fft_peak_fine_sampling_sync_0_0, rate, uhd_usrp_source_0, command] - [tempest_normalize_flow_0, '0', blocks_float_to_short_0, '0'] -- [tempest_sync_detector_0_0, '0', blocks_delay_0_0, '0'] - [uhd_usrp_source_0, '0', Keep_1_in_N_Frames_0, '0'] metadata: diff --git a/gr-tempest/examples/simulations/manual_simulated_tempest_TMDS_example.grc b/gr-tempest/examples/simulations/manual_simulated_tempest_TMDS_example.grc index 3e98098..f7dcf1a 100644 --- a/gr-tempest/examples/simulations/manual_simulated_tempest_TMDS_example.grc +++ b/gr-tempest/examples/simulations/manual_simulated_tempest_TMDS_example.grc @@ -41,14 +41,14 @@ blocks: bus_sink: false bus_source: false bus_structure: null - coordinate: [464, 148] + coordinate: [448, 148.0] rotation: 0 state: enabled - name: Hsize id: variable parameters: comment: '' - value: '1800' + value: '2200' states: bus_sink: false bus_source: false @@ -60,7 +60,7 @@ blocks: id: variable parameters: comment: '' - value: '1600' + value: '1920' states: bus_sink: false bus_source: false @@ -77,14 +77,14 @@ blocks: bus_sink: false bus_source: false bus_structure: null - coordinate: [384, 148] + coordinate: [352, 148.0] rotation: 0 state: enabled - name: Vsize id: variable parameters: comment: '' - value: '1000' + value: '1125' states: bus_sink: false bus_source: false @@ -96,7 +96,7 @@ blocks: id: variable parameters: comment: '' - value: '900' + value: '1080' states: bus_sink: false bus_source: false @@ -113,7 +113,7 @@ blocks: bus_sink: false bus_source: false bus_structure: null - coordinate: [304, 148] + coordinate: [280, 148.0] rotation: 0 state: enabled - name: epsilon_channel @@ -209,7 +209,7 @@ blocks: bus_sink: false bus_source: false bus_structure: null - coordinate: [232, 148] + coordinate: [216, 148.0] rotation: 0 state: enabled - name: interpolatedHsize @@ -285,8 +285,8 @@ blocks: orient: Qt.Horizontal rangeType: float start: '0' - step: 1e-4 - stop: 2e-1 + step: 1e-3 + stop: '2' value: '0' widget: counter_slider states: @@ -450,6 +450,22 @@ blocks: coordinate: [160, 408.0] rotation: 0 state: enabled +- name: blocks_complex_to_float_0 + id: blocks_complex_to_float + parameters: + affinity: '' + alias: '' + comment: '' + maxoutbuf: '0' + minoutbuf: '0' + vlen: '1' + states: + bus_sink: false + bus_source: false + bus_structure: null + coordinate: [824, 712.0] + rotation: 180 + state: true - name: blocks_complex_to_mag_0 id: blocks_complex_to_mag parameters: @@ -463,9 +479,9 @@ blocks: bus_sink: false bus_source: false bus_structure: null - coordinate: [824, 656.0] + coordinate: [832, 568.0] rotation: 180 - state: enabled + state: disabled - name: blocks_delay_0 id: blocks_delay parameters: @@ -736,7 +752,7 @@ blocks: alias: '' blanking: 'True' comment: '' - image_file: /home/emidan19/Desktop/deep-tempest/capturas/fine-tuning-images/Abolghasemi_Pay_Attention_-_Robustifying_a_Deep_Visuomotor_Policy_Through_Task-Focused_CVPR_2019_paper.png + image_file: /home/emidan19/Desktop/deep-tempest/drunet/notebooks/images/mattermost_screen.png maxoutbuf: '0' minoutbuf: '0' mode: '1' @@ -750,7 +766,7 @@ blocks: - name: tempest_buttonToFileSink_0 id: tempest_buttonToFileSink parameters: - Filename: /home/emidan19/Desktop/captura_simulada + Filename: /home/emidan19/Desktop/captura_simulada_shiftfix H_size: Hsize V_size: Vsize affinity: '' @@ -896,7 +912,9 @@ connections: - [binary_serializer_0_0_0, '0', blocks_add_xx_0, '2'] - [blocks_add_xx_0, '0', blocks_float_to_complex_0, '0'] - [blocks_add_xx_0, '0', interp_fir_filter_xxx_0, '0'] +- [blocks_complex_to_float_0, '0', tempest_normalize_flow_0, '0'] - [blocks_complex_to_mag_0, '0', tempest_normalize_flow_0, '0'] +- [blocks_delay_0, '0', blocks_complex_to_float_0, '0'] - [blocks_delay_0, '0', blocks_complex_to_mag_0, '0'] - [blocks_delay_0, '0', tempest_buttonToFileSink_0, '0'] - [blocks_float_to_complex_0, '0', blocks_throttle_0_0, '0'] diff --git a/gr-tempest/python/DTutils.py b/gr-tempest/python/DTutils.py index a99743b..2ef1c36 100644 --- a/gr-tempest/python/DTutils.py +++ b/gr-tempest/python/DTutils.py @@ -1,7 +1,8 @@ import numpy as np from matplotlib import pyplot as plt from scipy import signal -from scipy.fft import fft, ifft, fftshift +import cv2 as cv +from scipy.spatial import distance_matrix def autocorr(x): """Compute autocorrelation function of 1-D array @@ -463,3 +464,267 @@ def TMDS_serial(I): del(channel_list) return Iserials +def remove_outliers(I, radius=3, threshold=20): + """ + Replaces a pixel by the median of the pixels in the surrounding if it deviates from the median by more than a certain value (the threshold). + """ + + # Copy input + I_output = I.copy() + + # Apply median filter + I_median = cv.medianBlur(I,radius) + + # Replace with median value where difference with median exceedes threshold + where_replace = np.abs(I-I_median) > threshold + I_output[where_replace] = I_median[where_replace] + + return I_output + +def adjust_dynamic_range(I): + + I_output = I.astype('float32') + I_output = (I_output - I_output.min()) / (I_output.max() - I_output.min()) + I_output = 255 * I_output + return I_output.astype('uint8') + +def find_intersection(line1, line2): + """Finds the intersection of two lines given in Hesse normal form. + + Returns closest integer pixel locations. + See https://stackoverflow.com/a/383527/5087436 + """ + rho1, theta1 = line1 + rho2, theta2 = line2 + A = np.array([ + [np.cos(theta1), np.sin(theta1)], + [np.cos(theta2), np.sin(theta2)] + ]) + b = np.array([[rho1], [rho2]]) + x0, y0 = np.linalg.solve(A, b) + x0, y0 = int(np.round(x0)), int(np.round(y0)) + + # Return as row-column coordinates + return [y0, x0] +def apply_blanking_shift(I, h_active=1600, v_active=900, + h_blanking=200,v_blanking=100, + pad_len=300, debug=False): + """ + Find + """ + + if debug: + # Show original image + plt.figure(figsize=(12,10)) + plt.title(f'Original image ') + plt.imshow(I) + plt.axis('off') + plt.show() + + # Color to RGB + I_gray = cv.cvtColor(I,cv.COLOR_BGR2GRAY) + # I_gray = cv.medianBlur(I_gray,3) + I_gray = cv.GaussianBlur(I_gray,ksize=(5,5),sigmaX=3,sigmaY=3) + # I_gray = cv.blur(I_gray,(3,3)) + + # Wrap-padding image to get whole blanking pattern + pad_size_gray = ((pad_len,0),(pad_len,0)) + pad_size = ((pad_len,0),(pad_len,0),(0,0)) + I_gray = np.pad(I_gray,pad_size_gray,'wrap') + I = np.pad(I,pad_size,'wrap') + + # Edge-detector with Canny filter with empirical parameters + I_edge = cv.Canny(I_gray,40,50,apertureSize = 3) # sin blur + + + if debug: + # Show padded image and edges + plt.figure() + plt.title(f'Original image wrap-padded {pad_len} pixels up and left sided') + plt.imshow(I) + plt.axis('off') + plt.show() + plt.figure() + plt.title('Canny edged image') + plt.imshow(I_edge,cmap='gray') + plt.axis('off') + plt.show() + + # Hough Transform resolution and minimum votes threshold + theta_step = np.pi/2 + rho_step = 1 + # votes_thrs = 255 # sin blur + votes_thrs = 80 + rho_max = np.sqrt(I_gray.shape[0]**2 + I_gray.shape[1]**2) + + # Find Hough Transform + lines = cv.HoughLines(I_edge, rho_step, theta_step, votes_thrs, None, 0, 0) + + if debug: + # Show detected lines + I_lines = I_gray.copy() + for line in lines: + rho = line[0][0] + theta = line[0][1] + a = np.cos(theta) + b = np.sin(theta) + x0 = a * rho + y0 = b * rho + pt1 = (int(x0 + rho_max*(-b)), int(y0 + rho_max*(a))) + pt2 = (int(x0 - rho_max*(-b)), int(y0 - rho_max*(a))) + + cv.line(I_lines, pt1, pt2, (255,0,0), 1, cv.LINE_AA) + + plt.figure() + plt.title('All detected lines') + plt.imshow(I_lines) + plt.axis('off') + plt.show() + + # Angle and rho arrays + lines_angles = lines[:,0,1] + lines_rhos = lines[:,0,0] + + # Find unique lines angles detected + unique_angles = np.unique(lines_angles) + blankings = [h_blanking, v_blanking] + + # Initiate blanking lines variable + blanking_lines = [] + + for angle, blanking in zip(unique_angles, blankings): + + # Keep lines with certain angle + angle_lines = lines[lines[:,0,1]==angle] + angle_lines = angle_lines[:,0,:] + + # Keep the pair with rho distance that equals blanking + # First, compute the distance matrix over all rhos: + rho_angle_lines = np.array(angle_lines[:,0]).reshape(-1,1) + + rho_distances = np.abs(np.abs(distance_matrix(rho_angle_lines, rho_angle_lines)) - blanking) + + # Find minimum index + pair_lines_idx = np.unravel_index(rho_distances.argmin(), rho_distances.shape) + + # Add blanking limit line + for line in pair_lines_idx: + blanking_lines.append(angle_lines[line]) + + # List to array + blanking_lines = np.array(blanking_lines) + + # Show blanking limit lines + if debug: + I_lines = I_gray.copy() + + for line in blanking_lines: + rho = line[0] + theta = line[1] + a = np.cos(theta) + b = np.sin(theta) + x0 = a * rho + y0 = b * rho + pt1 = (int(x0 + rho_max*(-b)), int(y0 + rho_max*(a))) + pt2 = (int(x0 - rho_max*(-b)), int(y0 - rho_max*(a))) + + cv.line(I_lines, pt1, pt2, (255,0,0), 3, cv.LINE_AA) + + plt.figure(figsize=(12,10)) + plt.title('Blanking limit lines') + plt.imshow(I_lines) + plt.axis('off') + plt.show() + + # Initialize top-left corner blanking lines to find intersection + # These lines statisfies to be the ones with bigger rho value + blanking_start = [] + unique_angles = np.unique(blanking_lines[:,1]) + for angulo in unique_angles: + angle_lines = blanking_lines[blanking_lines[:,1]==angulo] + max_rho_line = angle_lines[np.argmax(angle_lines[:,0])] + blanking_start.append(max_rho_line) + + if len(blanking_start)!=2: + return -1 + + if debug: + I_lines = I_gray.copy() + for line in blanking_start: + rho = line[0] + theta = line[1] + a = np.cos(theta) + b = np.sin(theta) + x0 = a * rho + y0 = b * rho + pt1 = (int(x0 + rho_max*(-b)), int(y0 + rho_max*(a))) + pt2 = (int(x0 - rho_max*(-b)), int(y0 - rho_max*(a))) + + cv.line(I_lines, pt1, pt2, (255,0,0), 3, cv.LINE_AA) + + plt.figure(figsize=(12,10)) + plt.title('Blanking start lines') + plt.imshow(I_lines) + plt.axis('off') + plt.show() + + # Find blanking start lines intersection coordinates + x_shift, y_shift = find_intersection(blanking_start[0],blanking_start[1]) + + # Adjust to active image only (remove all blanking) + I_shift = I[pad_len:,pad_len:] + I_shift = np.roll(I_shift, -x_shift+pad_len, axis=0) + I_shift = np.roll(I_shift, -y_shift+pad_len, axis=1) + I_shift = I_shift[:v_active,:h_active] + + if debug: + plt.figure(figsize=(12,10)) + plt.title('Blanking removed image') + plt.imshow(I_shift) + plt.axis('off') + plt.show() + + return I_shift + +def preprocess_raw_capture(I, h_active, v_active, + h_blanking, v_blanking, debug=False): + """ + Center raw captured image, filter noise and adjust the contrast + """ + + # Center image. Returns -1 if no sufficient lines where detected + # In the latter case, use image as is without centering + is_centered = True + I_shift_fix = apply_blanking_shift(I, + h_active=h_active, v_active=v_active, + h_blanking=h_blanking, v_blanking=v_blanking, + debug=debug + ) + if np.shape(I_shift_fix) == (): + I_shift_fix = I.copy() + is_centered = False + + # Remove outliers with median thresholding heuristic + # Default: radius=3, threshold=20 + I_no_outliers = remove_outliers(I_shift_fix) + + # Stretch dynamic range to [0,255] + I_out = adjust_dynamic_range(I_no_outliers) + + if debug: + plt.figure(figsize=(12,10)) + ax0 = plt.subplot(3,1,1) + ax0.imshow(I_shift_fix, interpolation='none') + ax0.set_title('Centered image'*is_centered + 'Image'*(~is_centered)) + ax0.axis('off') + ax1 = plt.subplot(3,1,2, sharex=ax0, sharey=ax0) + ax1.imshow(I_no_outliers, interpolation='none') + ax1.set_title('Outliers removed') + ax1.axis('off') + ax1 = plt.subplot(3,1,3, sharex=ax0, sharey=ax0) + ax1.imshow(I_out, interpolation='none') + ax1.set_title('Contrast adjusted') + ax1.axis('off') + plt.show() + + return I_out \ No newline at end of file diff --git a/gr-tempest/python/buttonToFileSink.py b/gr-tempest/python/buttonToFileSink.py index e359df7..3f60580 100644 --- a/gr-tempest/python/buttonToFileSink.py +++ b/gr-tempest/python/buttonToFileSink.py @@ -16,15 +16,16 @@ from PIL import Image ### Deep-TEMPEST imports import os -import math import argparse -import time -import random import torch import sys + +# Adding gr-tempest/python folder to system path +dtutils_path = '/home/emidan19/deep-tempest/gr-tempest/python' +sys.path.insert(0,dtutils_path) +from DTutils import preprocess_raw_capture, apply_blanking_shift + # Adding KAIR folder to the system path -# current_file_path = os.path.abspath(__file__) -# sys.path.insert(0, os.path.join(current_file_path,'../../KAIR/')) kair_path = '/home/emidan19/deep-tempest/KAIR' sys.path.insert(0,kair_path) from utils import utils_option as option @@ -109,32 +110,49 @@ class buttonToFileSink(gr.sync_block): self.message_port_register_in(pmt.intern("en")) #declare message port self.set_msg_handler(pmt.intern("en"), self.handle_msg) #declare handler for messages self.stream_image = [] # initialize list to apppend samples + + # TODO: fancy active-blanking resolution identification + self.V_active = (self.V_size==1125)*1080 + (self.V_size==1000)*900 + (self.V_size==750) *720 + (self.V_size==628) *600 + (self.V_size==525)*480 + self.H_active = (self.H_size==2200)*1920 + (self.H_size==1800)*1600 + (self.H_size==1650)*1280 + (self.H_size==1056)*800 + (self.H_size==800)*640 + + self.V_blanking = self.V_size - self.V_active + self.H_blanking = self.H_size - self.H_active + if enhance_image: # Load model self.model = load_enhancement_model() # Define inference function def inference (img): - # Preprocess image to network input - img_L = img[:,:,:2] + # Preprocess image to network input + # (correct shift, remove sparkle noise and to float tensor) + img_L = preprocess_raw_capture(img, h_active=self.H_active, v_active=self.V_active, + h_blanking=self.H_blanking, v_blanking=self.V_blanking) + img_L = img_L[:,:,:2] img_L = util.uint2single(img_L) img_L = util.single2tensor4(img_L) # Model inference on image img_E = self.model(img_L) img_E = util.tensor2uint(img_E) + return img_E self.inference = inference else: - # No enhancement. + # No enhancement. Correct shift def inference(img): - return img + img_out = apply_blanking_shift(img, h_active=self.H_active, v_active=self.V_active, + h_blanking=self.H_blanking, v_blanking=self.V_blanking) + # Check for any errors. If so, return original image + if np.shape(img_out) == (): + img_out = img.copy() + return img_out self.inference = inference - def work(self, input_items, output_items): # Don't process, just save samples - - self.available_samples = len(input_items[0]) #available samples at the input for read at this moment + def work(self, input_items, output_items): + # Don't process, just save available samples + self.available_samples = len(input_items[0]) if self.en == True: @@ -144,19 +162,6 @@ class buttonToFileSink(gr.sync_block): if len(self.stream_image)==self.num_samples:#or self.remaining2Save > 0: - # if self.remaining2Save > 0 and self.remaining2Save < self.available_samples: #remaining to save from last time, don't care if true was pressed or not - # self.savingSamples = self.remaining2Save - # self.remaining2Save = 0 #no more remaining after this - # elif self.remaining2Save > 0: #the remaining still are more than available - # self.savingSamples = self.available_samples #save whatever available - # self.remaining2Save = self.remaining2Save - self.savingSamples # refresh whatever left to save next time - # else: #not remaining from last time but pressed true - # if self.num_samples > self.available_samples: #too large for the available samples - # self.savingSamples = self.available_samples # save whatever available and wait for the remaining - # self.remaining2Save = self.num_samples - self.savingSamples #remaining to save - # else: - # self.savingSamples = self.num_samples # enough samples available when pressing button - # Save the number of samples calculated before self.save_samples() # Back to default