From 6b24ddff0244323a60bb53d766caadc136a55a6a Mon Sep 17 00:00:00 2001 From: Daniel Fekete Date: Mon, 1 May 2017 15:56:06 +0200 Subject: [PATCH] Automated tests: automatically include referenced libraries --- .gitignore | 3 + tools/test/guess_includes.py | 390 +++++++++++++++++++++++++++++++++++ tools/test/makefile | 10 + 3 files changed, 403 insertions(+) create mode 100644 .gitignore create mode 100644 tools/test/guess_includes.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..80abc34 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +#Ignore files created by automatic test script +*.ino.h +build-include.txt diff --git a/tools/test/guess_includes.py b/tools/test/guess_includes.py new file mode 100644 index 0000000..a76e95c --- /dev/null +++ b/tools/test/guess_includes.py @@ -0,0 +1,390 @@ +import os +import glob +import json +import re +import subprocess +from subprocess import call, check_output + +BOARD = "BluePill_F103C8" + +DIR_CORES = "../../STM32" +DIR_PROJECTS = "sources/arduino_projects" + +def find_examples(dir, list): + if not os.path.isdir(dir): + return + + subdirs = os.listdir(dir) + for subdir in subdirs: + if os.path.exists(dir + '/' + subdir + '/' + subdir + '.ino'): + list.append(dir + '/' + subdir) + + if os.path.exists(dir + '/' + subdir + '/' + subdir + '.pde'): + list.append(dir + '/' + subdir) + + find_examples(dir + '/' + subdir, list) + +def find_directories(): + dirs = [] + + DIR_LIBRARIES = DIR_CORES + "/libraries" + core_libs = os.listdir(DIR_LIBRARIES) + for core_lib in core_libs: + if os.path.exists(DIR_LIBRARIES + "/" + core_lib + '/src'): + dirs.append(DIR_LIBRARIES + "/" + core_lib + '/src') + else: + dirs.append(DIR_LIBRARIES + "/" + core_lib) + + find_examples(DIR_LIBRARIES + '/' + core_lib + '/examples', dirs) + + return dirs + + projects = os.listdir(DIR_PROJECTS) + for project in projects: + if os.path.exists(DIR_PROJECTS + '/' + project + '/src'): + dirs.append(DIR_PROJECTS + '/' + project + '/src') + else: + dirs.append(DIR_PROJECTS + '/' + project) + + if os.path.exists(DIR_PROJECTS + '/' + project + '/examples'): + examples = os.listdir(DIR_PROJECTS + '/' + project + '/examples') + + #TODO use glob.glob() + for example in examples: + if not os.path.isdir(DIR_PROJECTS + '/' + project + '/examples/' + example): + continue + if os.path.exists(DIR_PROJECTS + '/' + project + '/examples/' + example + '/' + example + '.ino'): + dirs.append(DIR_PROJECTS + '/' + project + '/examples/' + example) + + if os.path.exists(DIR_PROJECTS + '/' + project + '/examples/' + example + '/' + example + '.pde'): + dirs.append(DIR_PROJECTS + '/' + project + '/examples/' + example) + + subexamples = os.listdir(DIR_PROJECTS + '/' + project + '/examples/' + example) + for subexample in subexamples: + if not os.path.isdir(DIR_PROJECTS + '/' + project + '/examples/' + example + '/' + subexample): + continue + + if os.path.isdir(DIR_PROJECTS + '/' + project + '/examples/' + example + '/' + subexample) and os.path.exists(DIR_PROJECTS + '/' + project + '/examples/' + example + '/' + subexample + '/' + subexample + '.ino'): + dirs.append(DIR_PROJECTS + '/' + project + '/examples/' + example + '/' + subexample) + if os.path.isdir(DIR_PROJECTS + '/' + project + '/examples/' + example + '/' + subexample) and os.path.exists(DIR_PROJECTS + '/' + project + '/examples/' + example + '/' + subexample + '/' + subexample + '.pde'): + dirs.append(DIR_PROJECTS + '/' + project + '/examples/' + example + '/' + subexample) + + subsubexamples = os.listdir(DIR_PROJECTS + '/' + project + '/examples/' + example + '/' + subexample) + for subsubexample in subsubexamples: + if os.path.isdir(DIR_PROJECTS + '/' + project + '/examples/' + example + '/' + subexample + '/' + subsubexample) and os.path.exists(DIR_PROJECTS + '/' + project + '/examples/' + example + '/' + subexample + '/' + subsubexample + '/' + subsubexample + '.ino'): + dirs.append(DIR_PROJECTS + '/' + project + '/examples/' + example + '/' + subexample + '/' + subsubexample) + if os.path.isdir(DIR_PROJECTS + '/' + project + '/examples/' + example + '/' + subexample + '/' + subsubexample) and os.path.exists(DIR_PROJECTS + '/' + project + '/examples/' + example + '/' + subexample + '/' + subsubexample + '/' + subsubexample + '.pde'): + dirs.append(DIR_PROJECTS + '/' + project + '/examples/' + example + '/' + subexample + '/' + subsubexample) + + return dirs + +core_includes = os.listdir(DIR_CORES + "/cores/arduino") + + +def remove_all(lst, key): + return [val for val in lst if val!=key] + +def get_includes(filename): + file = open(filename, "r") + includes = [] + + if "Printoo" in filename: #TODO remove this when SoftwareSerial is available + return includes + + declarations = [] + + for line in file: + if '//' in line: + line = line[:line.index('//')] + + line = line.rstrip() + + declaration_found = re.search('^([a-zA-Z].* .*\\(.*\\).*){$', line) + if declaration_found: + func_decl = declaration_found.groups()[0] + ';' + if func_decl.startswith('SIGNAL'): + continue + if "::" in func_decl[:func_decl.index('(')]: + continue + if "." in func_decl[:func_decl.index('(')]: + continue + if 'void setup(' in func_decl: + continue + if 'void loop(' in func_decl: + continue + declarations.append(func_decl); + + if line.rstrip() == '{': + combined = previous_line.rstrip() + line + declaration_found = re.search('^([a-zA-Z].* .*.*\\(.*\\).*){$', combined) + if declaration_found: + func_decl = declaration_found.groups()[0] + ';' + if 'void setup(' in func_decl: + continue + if 'void loop(' in func_decl: + continue + declarations.append(func_decl); + + + found = re.search('include.*?"(.*)"', line) + if found: + includes.append(found.groups()[0]) + found = re.search('include.*?<(.*)>', line) + if found: + includes.append(found.groups()[0]) + + + previous_line = line + + + for ignore_include in core_includes: + includes = remove_all(includes, ignore_include) + includes = remove_all(includes, "string.h") + includes = remove_all(includes, "variant.h") + includes = remove_all(includes, "time.h") + + if filename.endswith('.ino') or filename.endswith('.pde'): + with open(filename + '.h', 'w') as file: + if declarations: + for include in includes: + if include.startswith('avr/'): + continue + + file.write('#include "' + include + '"\n'); + file.write("\n".join(declarations)) + + return includes + +dirs = find_directories(); + +header_to_dir = {} + +for dir in dirs: + if 'examples' in dir: + continue + #print dir + headers = [file for file in os.listdir(dir) if file.endswith(('.h', '.hpp'))] + for header in headers: + if header in header_to_dir: + pass + #print header + ": " + header_to_dir[header] + "=>" + dir + else: + header_to_dir[header] = dir + +def get_sources(dir, subdir): + sources = [subdir + file for file in os.listdir(dir) if file.endswith(('.h', '.hpp', '.c', '.cpp', '.ino', '.pde'))] + for file in os.listdir(dir): + if os.path.isdir(dir + '/' + file) and file != 'examples': + sources.extend(get_sources(dir + '/' + file, subdir + file + '/')) + return sources + +def create_includes(): + #print check_output(["find", ".", "-name", "'build-include-*.txt'", "-delete"], stderr=subprocess.STDOUT) + + for dir in dirs: + + sources = get_sources(dir, '') + + dirs_to_include = [] + + for source in sources: + includes = get_includes(dir + '/' + source) + + for include in includes: + skip = False + for s in sources: + if s == include or include.endswith('/' + s) or s.endswith('/' + include): + skip = True + + if skip: + continue + + if include in header_to_dir: + dirs_to_include.append(header_to_dir[include]) + + dirs_to_include = list(set(dirs_to_include)) + + if dir in dirs_to_include: + dirs_to_include.remove(dir) + + add = [] + + for dir_to_include in dirs_to_include: + if os.path.exists(dir_to_include + '/build-include.txt'): + with open(dir_to_include + '/build-include.txt', 'r') as file: + add.extend(file.readline().strip().split(" ")) + + dirs_to_include.extend(add) + + dirs_to_include = list(set(dirs_to_include)) + + with open(dir + '/build-include.txt', 'w') as file: + for d in dirs_to_include: + file.write(d + ' ') + +create_includes() + +exit() + +already_compiled = [] + +def make(directory): + if (directory in already_compiled): + return + print directory + + already_compiled.append(directory) + + #try: + with open(directory + '/build-include-' + ARCH + '.txt') as f: + dependencies= f.readline() + for dependency in dependencies.split(): + make(dependency) + #except: + # print 'DEPENDENCIES NOT CREATED' + + call(['make', 'clean', 'ARCH='+ARCH, 'BOARD='+BOARD, 'PROJECT='+directory]) + + try: + os.mkdir(directory+"/build-"+BOARD) + except OSError: + pass + + stdout = open(directory+"/build-"+BOARD+"/stdout.txt", "w") + stderr = open(directory+"/build-"+BOARD+"/stderr.txt", "w") + call(['make', 'ARCH='+ARCH, 'BOARD='+BOARD, 'PROJECT='+directory], stdout = stdout, stderr = stderr) + stdout.close() + stderr.close() + +#create_includes() + +#raise X + +for dir in dirs: + make(dir) +#make('sources/arduino_projects/Adafruit_GFX_Library-1.1.5') +#make('sources/arduino_projects/Adafruit_ILI9341-1.0.1/examples/graphicstest') +#make('sources/arduino_projects/LcdProgressBarDouble-1.0.4/examples/DoubleBarPot') +#make('sources/arduino_projects/RTClib-1.2.0') +#make('sources/arduino_projects/Arduino_GUI-1.6.11/examples/01.Basics/Blink') +#make('sources/arduino_projects/SD-1.0.6/src') +#make('sources/arduino_projects/SD-1.0.6/examples/listfiles') +#make('sources/arduino_projects/Cayenne-1.0.1') +#make('sources/arduino_projects/ArduinoJson-5.6.7') +#make('sources/arduino_projects/ArduinoJson-5.6.7/examples/JsonParserExample') +#make('sources/arduino_projects/EMoRo_2560-2.4.1/src') +#make('sources/arduino_projects/Kalman_Filter_Library-1.0.1/examples/MPU6050') +#make('sources/arduino_projects/SparkFun_Graphic_LCD_Serial_Backpack-1.0.1/examples/SparkFunSerialGraphicLCDDemo') +#make('sources/arduino_projects/DHT_sensor_library-1.2.3') +#make('sources/arduino_projects/DHT_sensor_library-1.2.3/examples/DHTtester') +#make('sources/arduino_projects/Adafruit_GPS_Library-1.0.1/examples/leo_locus_erase') +#make('sources/arduino_projects/DimSwitch-1.0.2/examples/DimSwitchTester-ESP-MQTT') +#make('sources/arduino_projects/MFRC522-1.1.8/examples/RFID-Cloner') +#make('sources/arduino_projects/U8glib-1.19.1/examples/HelloWorld') +#make('sources/arduino_projects/Adafruit_CC3000_Library-1.0.3/examples/HTTPServer') +#make('sources/arduino_projects/Adafruit_Fingerprint_Sensor_Library-1.0.0') +#make('sources/arduino_projects/LiquidCrystal_I2C-1.1.2/examples/HelloWorld') +#make('sources/arduino_projects/AccelStepper') + +#raise Error + +output = [] + +errors = {} + +for dir in dirs: + #print dir + error = False + if not os.path.exists(dir+"/build-"+BOARD+"/stderr.txt"): + continue + + with open(dir+"/build-"+BOARD+"/stderr.txt", "r") as file: + for line in file: + #Find the most relevant error line + if 'warning: changing start of section ' in line: + continue + if 'In function ' in line: + continue + if 'In constructor ' in line: + continue + if 'In member function ' in line: + continue + if 'In file included from ' in line: + continue + if 'from sources/' in line: + continue + if 'from ./sources/' in line: + continue + if 'from ' in line: + continue + + if not error: error = line + + #print "ERROR = ", error + if not error: + error = "SUCCESS" + + errors[dir] = error.strip() + +with open('cache/library_index.json') as file: + data = json.load(file) + +libraries = {} + +for library in data["libraries"]: + libraries[library["name"]] = library + +for name, library in libraries.iteritems(): + if "github.com" in library["website"]: + api = library["website"].replace("github.com", "api.github.com/repos") + api = api.replace("https", "http") + if api.endswith(".git"): + api = api[:-4] + download_file = "cache/github_api/" + api[len("http://api.github.com/repos")+1:].replace("/", ":") + + if os.path.exists(download_file): + with open(download_file) as file: + data = json.load(file) + libraries[name]["github_api"] = data + libraries[name]["sort"] = data["forks_count"] + data["watchers_count"] + data["stargazers_count"] + data["subscribers_count"] + libraries[name]["sort"] = data["stargazers_count"] + + zip_file = os.path.basename(library["url"]) + projects_folder = DIR_PROJECTS + "/" + zip_file[:-4] + if projects_folder in errors: + libraries[name]["error"] = errors[projects_folder] + elif (projects_folder + "/src") in errors: + libraries[name]["error"] = errors[projects_folder + "/src"] + else: + libraries[name]["error"] = "NOT FOUND" + + libraries[name]['example_errors'] = {} + libraries[name]['projects_folder'] = projects_folder + + for error_folder in errors: + if 'examples' in error_folder and error_folder.startswith(projects_folder): + #print projects_folder, error_folder, errors[error_folder] + libraries[name]['example_errors'][error_folder] = errors[error_folder] + + #if (libraries[name]['author'] == 'Arduino'): + # libraries[name]["sort"] = 10000 + + + +def sort_library(a): + if "sort" in libraries[a]: + return libraries[a]["sort"] + else: + return 0 + +library_names = sorted(libraries, key=sort_library, reverse=True) + +for name in library_names: + if 'sort' in libraries[name]: + pass + #print name, libraries[name]["sort"], libraries[name]["website"], libraries[name]["projects_folder"], libraries[name]["error"], libraries[name]["example_errors"] + +with open('result.json', 'w') as file: + file.write(json.dumps({'libraries': libraries, 'library_names': library_names})) + + diff --git a/tools/test/makefile b/tools/test/makefile index 89e2c9d..6e75a02 100644 --- a/tools/test/makefile +++ b/tools/test/makefile @@ -10,6 +10,11 @@ PATH_BUILD_PROJECT = $(PATH_BUILD_ROOT)/$(PROJECT)/ include makefiles/$(VARIANT) +### Included libraries + +cat := $(if $(filter $(OS),Windows_NT),type,cat) +LIBRARIES = $(shell $(cat) $(subst /,\\,$(PATH_PROJECT)/build-include.txt)) + #### Source files PROJECT_SOURCES_C += $(wildcard $(PATH_PROJECT)*.c) @@ -39,6 +44,10 @@ OBJECTS += $(patsubst $(PATH_CORE)%,$(PATH_BUILD_CORE)%,$(CORE_SOURCES_S:.S=.s.o OBJECTS += $(patsubst $(PATH_VARIANT)%,$(PATH_BUILD_CORE)%,$(VARIANT_SOURCES_C:.c=.c.o)) OBJECTS += $(patsubst $(PATH_VARIANT)%,$(PATH_BUILD_CORE)%,$(VARIANT_SOURCES_CPP:.cpp=.cpp.o)) + +LIBRARY_OBJ_DIRS += $(patsubst $(PATH_ROOT)libraries%,$(PATH_BUILD_ROOT)%,$(LIBRARIES)) +OBJECTS += $(foreach dir,$(LIBRARY_OBJ_DIRS),$(wildcard $(dir)/*.o)) + ifneq ($(wildcard $(PATH_PROJECT)/*.ino),) ELF = $(PATH_BUILD_CORE)build.elf BIN = $(PATH_BUILD_CORE)build.bin @@ -55,6 +64,7 @@ INCLUDES += "-I$(PATH_ROOT)system/STM32F1/HAL_Inc" INCLUDES += "-I$(PATH_ROOT)system/STM32F1/HAL_Src" INCLUDES += "-I$(PATH_ROOT)system/STM32F1/stm32_chip" INCLUDES += "-I$(PATH_VARIANT)" +INCLUDES += $(LIBRARIES:%=-I%) all: $(OBJECTS) $(BIN) $(info BUILD SUCCESSFUL)