From 2e6bf309e0715e23b9b26743502eac6e14503da3 Mon Sep 17 00:00:00 2001 From: David Holdeman Date: Tue, 14 Jul 2020 09:28:26 -0500 Subject: [PATCH] Fix #1590 (#1602) * Add checkout * remove empty with statement * Actually we do need push depth * convert gen_upload_docs.bat * convert run.bat * forgot to add run.sh * convert gen_upload_docs.bat * remove checking from bats * change config * fix path * fix path again * fix file extension * remove escapes * added missing quotation and escapes * convert build_current_bundle * convert clean_bootloader * convert compile_bootloader_discovery * convert compile_bootloader * add env var to skip upload * remove checking from compile.bat * working on push.yaml * finish sorting * progress * rename push to builds * rename push to builds * Add skip upload env var * fix pull_request name * fixed typo and changed name * fixed indentation and runs-on * try different env var setup * add nucleos and switch to underscore * add quotes * remove boards that were commented * remove boards that were commented * fix project board name * fixed variable name * add ini-file and console-settings * try installing gcc-multilib first * Install g++-multilib as well * remove unnecessary export * fixed exit numbers * change to different env setup * add simulator and console compilation to board bundles * changed OS and added linux simulator build * add checkout and missing colon * add gcc-multilib installes * Install mingw * fixed install * remove mingw from linux sim and call compile directly * change directory * moved envs to command * debug ls * debug ls * set ftp vars * add makefile debug info * removed batchecks * redirect stdout to easier see orrs * change name * remove redirect, add file inpection * Add conditional project name * Try software upgrade * missing char * update ine files * try quotes * add escapes * add quotes * add escapes and quotes * remove quotes and escapes * remove software upgrade * ignore test 2 * update apt * like this then * forgot import * fix case * remove RUSEFI_SKIP_UPLOAD and check RUSEFI_FTP_SERVER instead * break workflow back apart * upload simulator artifact * fixed operator * update apt * change paths * Update build-firmware.yaml * move update bundle to artifacts * fix var name * Install ncftp * set env vars conditionally * remove check so zips are created * try things the github way and copy to primary * re-enable test * missing space * use bash * use bash * add workflow * add check before upload * install kicad * switch firmware to bash * switch unit_testts to bash * switch jenkins to bash * fix path * pwd * checkout submodules * add ppa for kicad 5.1 * install xvfb and set DISPLAY * use xvfb * use python3 and add NO_DISPLAY var * fetch all * remove blob * undo unintentional changes * reset accidentally commited config.xmlu * remove submodule * fix path * Only call one board * make simulator and console conditional * break away bundle packaging * pass bundle name * switch to bash * make primary bundle conditional * break away packaging for primary and pass config names * Add default name to unit tests * add check to skip config * add property to skip config, and fix check * switch primary bundle to still build * switch to bash * move default config to its own script * add exit 0 * go back to gen_config running all boards * remove TS stuf8f * remove TS stuff * switch to gen_config_board * switch to bash * add kinetis back * add check for kinetis * rename compile_and_upload * add error checking * Revert "add error checking" This reverts commit f6c9b7f2521f9c28ea150da8e733c593a59596a1. * Add readme Co-authored-by: David Holdeman --- .github/workflows/build-firmware.yaml | 39 ++++++++++++--- .github/workflows/build-unit-tests.yaml | 4 +- .github/workflows/gen-configs.yaml | 4 +- .github/workflows/readme.md | 27 ++++++++++ firmware/gen_config.sh | 50 ++----------------- firmware/gen_config_board.sh | 9 ---- firmware/gen_config_default.sh | 34 +++++++++++++ .../{compile_and_upload.sh => compile.sh} | 2 - .../compile_and_upload.bat | 3 +- .../compile_other_versions/prepare_bundle.sh | 2 +- 10 files changed, 104 insertions(+), 70 deletions(-) create mode 100644 .github/workflows/readme.md create mode 100644 firmware/gen_config_default.sh rename misc/jenkins/compile_other_versions/{compile_and_upload.sh => compile.sh} (92%) diff --git a/.github/workflows/build-firmware.yaml b/.github/workflows/build-firmware.yaml index 3778fe1a0c..d3e4f72105 100644 --- a/.github/workflows/build-firmware.yaml +++ b/.github/workflows/build-firmware.yaml @@ -55,11 +55,13 @@ jobs: folder: nucleo_f746 ini-file: no console-settings: firmware/config/boards/nucleo_f767/rusefi_console_properties.xml + skip-config: yes - build-target: stm32f767_nucleo folder: nucleo_f767 ini-file: no console-settings: firmware/config/boards/nucleo_f767/rusefi_console_properties.xml + skip-config: yes steps: - uses: actions/checkout@v1 @@ -67,12 +69,20 @@ jobs: submodules: recursive - name: Generate Configs + if: ${{ matrix.skip-config != 'yes' }} working-directory: ./firmware/ - run: sh gen_config.sh + run: | + if [ "${{ matrix.build-target }}" = "kinetis" ]; then + cd config/boards/kinetis/config + bash gen_config.sh + [ $? -eq 0 ] || { echo "ERROR generating board kinetis kin"; exit 1; } + else + bash gen_config_board.sh ${{matrix.folder}} ${{matrix.build-target}} + fi - name: Generate Live Documentation working-directory: ./firmware/ - run: sh gen_live_documentation.sh + run: bash gen_live_documentation.sh # Build machines don't have arm-none-eabi gcc, so let's download it and put it on the path - name: Download & Install GCC @@ -100,23 +110,31 @@ jobs: # Build rusEFI console - name: Build console - run: sh misc/jenkins/build_java_console.sh + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + run: bash misc/jenkins/build_java_console.sh # Build the simulator - name: Build simulator - run: OS="Windows_NT" sh misc/jenkins/build_simulator.sh + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + run: OS="Windows_NT" bash misc/jenkins/build_simulator.sh # Build the firmware! - name: Build Firmware - run: sh misc/jenkins/compile_other_versions/compile_and_upload.sh ${{matrix.folder}} ${{matrix.build-target}} ${{matrix.ini-file}} ${{matrix.console-settings}} + run: bash misc/jenkins/compile_other_versions/compile.sh ${{matrix.folder}} ${{matrix.build-target}} ${{matrix.ini-file}} ${{matrix.console-settings}} + + - name: Package Bundle + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + run: bash misc/jenkins/compile_other_versions/prepare_bundle.sh ${{matrix.build-target}} - name: Upload bundle + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} uses: actions/upload-artifact@v2 with: name: rusefi_bundle_${{matrix.build-target}}.zip path: ./artifacts/rusefi_bundle_${{matrix.build-target}}.zip - name: Upload autoupdate bundle + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} uses: actions/upload-artifact@v2 with: name: rusefi_bundle_${{matrix.build-target}}_autoupdate.zip @@ -131,11 +149,11 @@ jobs: - name: Generate Configs working-directory: ./firmware/ - run: sh gen_config.sh + run: bash gen_config_default.sh - name: Generate Live Documentation working-directory: ./firmware/ - run: sh gen_live_documentation.sh + run: bash gen_live_documentation.sh # Build machines don't have arm-none-eabi gcc, so let's download it and put it on the path - name: Download & Install GCC @@ -162,9 +180,14 @@ jobs: fi - name: Build Primary Bundle - run: OS="Windows_NT" sh misc/jenkins/functional_test_and_build_bundle/build_current_bundle.sh + run: OS="Windows_NT" bash misc/jenkins/functional_test_and_build_bundle/build_current_bundle.sh + + - name: Package Bundle + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + run: bash misc/jenkins/compile_other_versions/prepare_bundle.sh default - name: Upload primary bundle + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} uses: actions/upload-artifact@v2 with: name: rusefi_bundle.zip diff --git a/.github/workflows/build-unit-tests.yaml b/.github/workflows/build-unit-tests.yaml index 51489cfacd..9c53e9caa4 100644 --- a/.github/workflows/build-unit-tests.yaml +++ b/.github/workflows/build-unit-tests.yaml @@ -13,11 +13,11 @@ jobs: - name: Generate Configs working-directory: ./firmware/ - run: sh gen_config.sh + run: bash gen_config_default.sh - name: Generate Live Documentation working-directory: ./firmware/ - run: sh gen_live_documentation.sh + run: bash gen_live_documentation.sh - name: Build Tests working-directory: ./unit_tests/ diff --git a/.github/workflows/gen-configs.yaml b/.github/workflows/gen-configs.yaml index 262db445f0..2d6508b150 100644 --- a/.github/workflows/gen-configs.yaml +++ b/.github/workflows/gen-configs.yaml @@ -19,7 +19,7 @@ jobs: - name: Generate Configs working-directory: ./firmware/ - run: sh gen_config.sh + run: bash gen_config.sh - name: Upload .ini files working-directory: ./firmware/tunerstudio/generated @@ -28,7 +28,7 @@ jobs: - name: Generate Live Documentation working-directory: ./firmware/ - run: sh gen_live_documentation.sh + run: bash gen_live_documentation.sh # Commit changes and catch the error that occurs if nothing has been changed (without catching other errors) - name: Commit changes diff --git a/.github/workflows/readme.md b/.github/workflows/readme.md new file mode 100644 index 0000000000..84a34cf2da --- /dev/null +++ b/.github/workflows/readme.md @@ -0,0 +1,27 @@ +Here is a diagram of which configure scripts are used for which workflows. +This is not a complete diagram of all workflows, nor does it show everything that these jobs do. + +``` + Jenkins + + + v + run.bat + +------------+-----------------------+ + | | v + | | compile_and_upload.bat + | | +--------------+ +------------------------------+ + | | | v v v + | | | gen_config compile.sh + | | | +--------+-------------------+ + + v v | v v v v + Sim Console | ConfigKinetis ConfigDefault ConfigBoard Firmware Package + ^ ^ | ^ ^ ^ ^ ^ ^ ^ ^ + | | | | | | | | | | | + | | | | | | | | | | | + | | | | | | | +------+ | | + | | | | | | | | | | + +---+--------+-------------+-------------+--------------+--------------+-----------+ + | | | | + + + + + +Firmware Push Gen Config Unit Tests Firmware PR +``` diff --git a/firmware/gen_config.sh b/firmware/gen_config.sh index 84e78e4ed7..17ceb6c090 100644 --- a/firmware/gen_config.sh +++ b/firmware/gen_config.sh @@ -1,54 +1,14 @@ #!/bin/bash -#set -x -#TS_PATH="/home//TunerStudioProjects/" +echo "This script reads rusefi_config.txt and produces firmware persistent configuration headers" +echo "The storage section of rusefi.ini is updated as well" -echo "This batch files reads rusefi_config.txt and produses firmware persistent configuration headers" -echo "the storage section of rusefi.ini is updated as well" - -rm gen_config.log -rm gen_config_board.log - -mkdir build - -bash gen_signature.sh all - -java -DSystemOut.name=gen_config \ - -Drusefi.generator.lazyfile.enabled=true \ - -jar ../java_tools/ConfigDefinition.jar \ - -definition integration/rusefi_config.txt \ - -romraider integration \ - -ts_destination tunerstudio \ - -ts_output_name generated/rusefi.ini \ - -cache . \ - -cache_zip_file tunerstudio/generated/cache.zip \ - -with_c_defines false \ - -initialize_to_zero false \ - -tool gen_config.sh \ - -c_defines controllers/generated/rusefi_generated.h \ - -firing_order controllers/algo/firing_order.h \ - -c_destination controllers/generated/engine_configuration_generated_structures.h \ - -c_fsio_constants controllers/generated/fsio_enums_generated.def \ - -c_fsio_getters controllers/generated/fsio_getters.def \ - -c_fsio_names controllers/generated/fsio_names.def \ - -c_fsio_strings controllers/generated/fsio_strings.def \ - -java_destination ../java_console/models/src/main/java/com/rusefi/config/generated/Fields.java \ - -signature tunerstudio/generated/signature_all.txt \ - -signature_destination controllers/generated/signature_all.h \ - -romraider_destination ../java_console/rusefi.xml +rm -f gen_config.log +rm -f gen_config_board.log +bash gen_config_default.sh [ $? -eq 0 ] || { echo "ERROR generating default"; exit 1; } -if [ -z "${TS_PATH}" ]; then - echo "TS_PATH not defined" - # it's nice to have default location - TS_PATH="${HOMEDRIVE}${HOMEPATH}\Documents\TunerStudioProjects" -fi - -echo "This would automatically copy latest file to 'dev' TS projects to ${TS_PATH}" -cp -v tunerstudio/generated/rusefi.ini $TS_PATH/dev/projectCfg/mainController.ini -cp -v tunerstudio/generated/rusefi_mre_f4.ini $TS_PATH/mre_f4/projectCfg/mainController.ini - for BOARD in "microrusefi mre_f7" "microrusefi mre_f4" "frankenso frankenso_na6" "prometheus prometheus_469" "prometheus prometheus_405" "proteus proteus_f7" "proteus proteus_f4"; do BOARD_NAME="${BOARD% *}" BOARD_SHORT_NAME="${BOARD#* }" diff --git a/firmware/gen_config_board.sh b/firmware/gen_config_board.sh index 32015aa8ff..5bbbf230a9 100644 --- a/firmware/gen_config_board.sh +++ b/firmware/gen_config_board.sh @@ -48,13 +48,4 @@ java -DSystemOut.name=gen_config_board \ [ $? -eq 0 ] || { echo "ERROR generating TunerStudio config for ${BOARDNAME}"; exit 1; } -if [ -z "${TS_PATH}" ]; then - echo "TS_PATH not defined" -else - if [ -d "${TS_PATH}/dev_${BOARDNAME}/" ]; then - echo "This would automatically copy latest file to 'dev_${BOARDNAME}' TS project $TS_PATH" - cp -v tunerstudio/generated/rusefi_${BOARDNAME}.ini ${TS_PATH}/dev_${BOARDNAME}/projectCfg/mainController.ini - fi -fi - exit 0 diff --git a/firmware/gen_config_default.sh b/firmware/gen_config_default.sh new file mode 100644 index 0000000000..176e815798 --- /dev/null +++ b/firmware/gen_config_default.sh @@ -0,0 +1,34 @@ +mkdir build + +rm -f gen_config.log +rm -f gen_config_board.log + +bash gen_signature.sh all + +java -DSystemOut.name=gen_config \ + -Drusefi.generator.lazyfile.enabled=true \ + -jar ../java_tools/ConfigDefinition.jar \ + -definition integration/rusefi_config.txt \ + -romraider integration \ + -ts_destination tunerstudio \ + -ts_output_name generated/rusefi.ini \ + -cache . \ + -cache_zip_file tunerstudio/generated/cache.zip \ + -with_c_defines false \ + -initialize_to_zero false \ + -tool gen_config.sh \ + -c_defines controllers/generated/rusefi_generated.h \ + -firing_order controllers/algo/firing_order.h \ + -c_destination controllers/generated/engine_configuration_generated_structures.h \ + -c_fsio_constants controllers/generated/fsio_enums_generated.def \ + -c_fsio_getters controllers/generated/fsio_getters.def \ + -c_fsio_names controllers/generated/fsio_names.def \ + -c_fsio_strings controllers/generated/fsio_strings.def \ + -java_destination ../java_console/models/src/main/java/com/rusefi/config/generated/Fields.java \ + -signature tunerstudio/generated/signature_all.txt \ + -signature_destination controllers/generated/signature_all.h \ + -romraider_destination ../java_console/rusefi.xml + +[ $? -eq 0 ] || { echo "ERROR generating default"; exit 1; } + +exit 0 diff --git a/misc/jenkins/compile_other_versions/compile_and_upload.sh b/misc/jenkins/compile_other_versions/compile.sh similarity index 92% rename from misc/jenkins/compile_other_versions/compile_and_upload.sh rename to misc/jenkins/compile_other_versions/compile.sh index 40e4061745..9a0a440186 100644 --- a/misc/jenkins/compile_other_versions/compile_and_upload.sh +++ b/misc/jenkins/compile_other_versions/compile.sh @@ -36,6 +36,4 @@ cd ../../../.. # We are back at root rusEfi folder pwd -bash misc/jenkins/compile_other_versions/prepare_bundle.sh - echo "exiting $SCRIPT_NAME" diff --git a/misc/jenkins/compile_other_versions/compile_and_upload.bat b/misc/jenkins/compile_other_versions/compile_and_upload.bat index 0d2a67d114..ade38a12ec 100644 --- a/misc/jenkins/compile_other_versions/compile_and_upload.bat +++ b/misc/jenkins/compile_other_versions/compile_and_upload.bat @@ -1,2 +1,3 @@ @echo off -sh.exe misc\jenkins\compile_other_versions\compile_and_upload.sh %1 %2 %3 %4 +bash.exe misc\jenkins\compile_other_versions\compile.sh %1 %2 %3 %4 +bash.exe misc\jenkins\compile_other_versions\prepare_bundle.sh %1 %2 %3 %4 diff --git a/misc/jenkins/compile_other_versions/prepare_bundle.sh b/misc/jenkins/compile_other_versions/prepare_bundle.sh index 0344212da0..8594feb0d3 100644 --- a/misc/jenkins/compile_other_versions/prepare_bundle.sh +++ b/misc/jenkins/compile_other_versions/prepare_bundle.sh @@ -6,7 +6,7 @@ echo "RUSEFI_BUILD_FTP_USER=$RUSEFI_BUILD_FTP_USER" TIMESTAMP=$(date "+%Y%m%d_%H%M%S") -export FOLDER="temp/snapshot_${TIMESTAMP}_${BUNDLE_NAME}_rusefi" +export FOLDER="temp/snapshot_${TIMESTAMP}_${2}_rusefi" export BUNDLE_FULL_NAME="rusefi_bundle_$BUNDLE_NAME"