diff --git a/bin/check_all.sh b/bin/check_all.sh index 7de8de1..157c3e7 100755 --- a/bin/check_all.sh +++ b/bin/check_all.sh @@ -3,13 +3,13 @@ echo "Checking the environment..." unameOut="$(uname -s)" case "${unameOut}" in - Linux*) machine=linux;; - Darwin*) machine=mac;; - CYGWIN*) machine=cygwin;; - MSYS*) machine=msys;; - MINGW32*) machine=mingw32;; - MINGW64*) machine=mingw64;; - *) machine=unknown;; + Linux*) machine=linux;; + Darwin*) machine=mac;; + CYGWIN*) machine=cygwin;; + MSYS*) machine=msys;; + MINGW32*) machine=mingw32;; + MINGW64*) machine=mingw64;; + *) machine=unknown;; esac if [ "${machine}" = "unknown" ] ; then echo "* Warning! Unknown environment: ${unameOut}" @@ -43,17 +43,17 @@ function install_package { # we give it one more chance and try to download the installer echo "Do you want to download the cygwin package manager (apt-cyg) and install the required utilities? (Press 1 or 2)" select yn in "Yes" "No"; do - case $yn in - Yes ) - url="rawgit.com/transcode-open/apt-cyg/master/apt-cyg" - dst="/tmp/apt-cyg" + case $yn in + Yes ) + url="rawgit.com/transcode-open/apt-cyg/master/apt-cyg" + dst="/tmp/apt-cyg" download_url $url $dst - install $dst /bin - rm $dst - break;; - No ) + install $dst /bin + rm $dst + break;; + No ) echo "Please install it manually using you package manager!" >&2 - exit 1; + exit 1; esac done else @@ -63,19 +63,19 @@ function install_package { elif [ "${machine}" = "msys" ] ; then if [ ! -x "$(command -v pacman)" ] ; then echo "Cannot detect pacman manager. Please install it manually using you package manager!" >&2 - exit 1; + exit 1; fi fi # now install echo "Do you want to install '$1' now? (Press 1 or 2)" select yn in "Yes" "No"; do - case $yn in + case $yn in Yes ) - break;; + break;; No ) echo "Please install it manually using your package manager!" >&2 - exit 1; + exit 1; esac done @@ -89,23 +89,23 @@ function install_package { sudo apt update if ! sudo apt-get install $1; then echo "Error! Cannot install the package $1. Please install it manually using your package manager!" >&2 - exit 1; - fi + exit 1; + fi fi fi } echo "Checking the Python version..." -# check python version - should be 2.x ONLY -python_bin="python2.7" +# check python version - should be 3.x ONLY +python_bin="python3" while true; do python_ver=$($python_bin -V 2>&1 | grep -Po '(?<=Python )(.+)') - if [[ -z "$python_ver" ]] || [[ ! $python_ver =~ ^2\.7.* ]] ; then - echo "Error! Python 2.7.x is required. It should be installed and added to the PATH!" - install_package python2 - if [ "${machine}" = "linux" ] ; then - install_package python2-dev - fi + if [[ -z "$python_ver" ]] || [[ ! $python_ver =~ ^3\.[56789].* ]] ; then + echo "Error! Python 3.5 or later is required. It should be installed and added to the PATH!" + install_package python2 + if [ "${machine}" = "linux" ] ; then + install_package python2-dev + fi else break fi @@ -116,10 +116,10 @@ function check_library { echo "* Checking $1..." while true; do if ! command -v pkg-config >/dev/null 2>&1 ; then - echo "* Missing pkg-config" + echo "* Missing pkg-config" echo "Do you want to download and install it now? (Press 1 or 2)" select yn in "Yes" "No"; do - case $yn in + case $yn in Yes ) install_package pkg-config; break;; No ) exit 1; esac @@ -137,35 +137,35 @@ function check_library { done } -function install_pip2 { - pip2installer="https://bootstrap.pypa.io/pip/2.7/get-pip.py" +function install_pip3 { + pip3installer="https://bootstrap.pypa.io/pip/3.6/get-pip.py" dst="/tmp/get-pip.py" - download_url $pip2installer $dst + download_url $pip3installer $dst $python_bin $dst rm $dst } -function pip2_install_module { - # check if pip2 works +function pip3_install_module { + # check if pip3 works pip_bin="$python_bin -m pip" while true; do - pip2v=$($pip_bin --version 2>&1 | grep -Po '(pip [0-9]+\.[0-9]+.*)') - if [[ -z "$pip2v" ]] ; then - echo "* Missing pip2" + pip3v=$($pip_bin --version 2>&1 | grep -Po '(pip [0-9]+\.[0-9]+.*)') + if [[ -z "$pip3v" ]] ; then + echo "* Missing pip3" if [ "${machine}" = "linux" ] ; then - echo "Do you want to download and install it now? (Press 1 or 2)" - select yn in "Yes" "No"; do - case $yn in - Yes ) install_pip2; break;; - No ) exit 1; - esac - done - else - install_package pip2 - fi - else - break - fi + echo "Do you want to download and install it now? (Press 1 or 2)" + select yn in "Yes" "No"; do + case $yn in + Yes ) install_pip3; break;; + No ) exit 1; + esac + done + else + install_package pip3 + fi + else + break + fi done # check if gnu compiler works (needed by some modules) @@ -173,14 +173,14 @@ function pip2_install_module { while true; do gccv=$($gcc_bin -v 2>&1 | grep -Po '(version\s+[0-9]+\.[0-9]+.*)') if [[ -z "$gccv" ]] ; then - echo "* Missing gcc compiler" - install_package gcc - else - break - fi + echo "* Missing gcc compiler" + install_package gcc + else + break + fi done - echo "* Installing python module $1 using $pip2v and gcc $gccv..." + echo "* Installing python module $1 using $pip3v and gcc $gccv..." $pip_bin install $pymodule } @@ -192,8 +192,8 @@ git_ver=0 while true; do git_ver=$($git_bin version 2>&1 | grep -Po '(version [0-9]+\.[0-9]+.*)') if [[ -z "$git_ver" ]] ; then - echo "Error! Git not found! We need it to update submodules." - install_package git + echo "Error! Git not found! We need it to update submodules." + install_package git else break fi @@ -201,11 +201,16 @@ done echo "* Git $git_ver detected!" echo "Updating git submodules for scripts..." -git submodule update --init -- bin/gerbmerge bin/python-combine-pdfs bin/InteractiveHtmlBom bin/pcb-tools +#git submodule update --init -- bin/gerbmerge bin/python-combine-pdfs bin/InteractiveHtmlBom bin/pcb-tools echo "Checking the Python modules..." declare -A modules modules[simpleparse]=simpleparse +modules[moderngl]=ModernGL +modules[PIL]=Pillow +modules[pyrr]=Pyrr +modules[numpy]=NumPy +modules[vrml.vrml97]=PyVRML97 modules[contextlib2]=contextlib2 modules[PyPDF2]=PyPDF2 modules[gerber]=gerber @@ -219,13 +224,13 @@ for module in "${!modules[@]}"; do while true; do $python_bin -c "import sys, pkgutil; sys.path.append('./bin/pcb-tools'); sys.exit(0 if (pkgutil.find_loader('$module')) else 1)" if [ $? -eq 0 ]; then - echo "* Checking Python module '$pymodule': OK" - break + echo "* Checking Python module '$pymodule': OK" + break else - echo "* Checking Python module '$pymodule': ERROR!" - echo " Python module '$pymodule' is required and NOT found!" - - # some modules have dependencies + echo "* Checking Python module '$pymodule': ERROR!" + echo " Python module '$pymodule' is required and NOT found!" + + # some modules have dependencies if [ "$pymodule" = "cairocffi" ]; then if [ "${machine}" = "linux" ] ; then devname="dev" @@ -236,15 +241,15 @@ for module in "${!modules[@]}"; do check_library cairo cairo libcairo-${devname} fi - if [ ]; then - echo "Please use 'pip2 install $pymodule' to install it manually!" - exit 1; - else + if [ ]; then + echo "Please use 'pip3 install $pymodule' to install it manually!" + exit 1; + else echo "Do you want to download and install it now? (Press 1 or 2)" select yn in "Yes" "No"; do - case $yn in - Yes ) pip2_install_module $pymodule; break;; - No ) exit 1; + case $yn in + Yes ) pip3_install_module $pymodule; break;; + No ) exit 1; esac done fi @@ -252,60 +257,6 @@ for module in "${!modules[@]}"; do done done -echo "Checking if Node.js is installed..." -node_bin="node" -node_ver=0 -while true; do - node_ver=$($node_bin -v 2>&1 | grep -Po '(v[0-9]+.*)') - if [[ -z "$node_ver" ]] ; then - echo "Error! This script requires Node.Js installed in PATH!" - if [ "${machine}" = "linux" ] ; then - install_package nodejs - else - echo "Please download and install it from here: https://nodejs.org/en/download/" - exit 1 - fi - else - break - fi -done - -echo "* Node.js $node_ver detected!" - -echo "Checking npm..." -npm_bin="npm" -npm_ver=0 -while true; do - npm_ver=$($npm_bin -v 2>&1 | grep -Po '([0-9]+\.[0-9]+.*)') - if [[ -z "$npm_ver" ]] ; then - echo "Error! NPM not found! We need it to check Node.Js packages." - install_package npm - else - break - fi -done -echo "* NPM $npm_ver detected!" - - -echo "Checking Node.js packages..." -pushd ./bin/render_vrml > /dev/null -for package in 'puppeteer' 'pngjs' 'fs' 'zlib'; do - if [ `npm list --depth=0 | grep -c "${package}@"` -eq 1 ]; then - echo "* Checking Node.js module '$package': OK" - else - echo "* Checking Node.js module '$package': ERROR!" - echo " The module '$package' is required and NOT found! Please use 'npm install $package' to install it" - echo "Do you want to download and install it now? (Press 1 or 2)" - select yn in "Yes" "No"; do - case $yn in - Yes ) npm install $package --no-shrinkwrap; break;; - No ) exit 1; - esac - done - fi -done -popd > /dev/null - echo "All checks done!" exit 0 diff --git a/bin/create_3d_components.py b/bin/create_3d_components.py index 419b36e..50cf0c5 100644 --- a/bin/create_3d_components.py +++ b/bin/create_3d_components.py @@ -23,7 +23,7 @@ config.read(mergeBoardFile) # read place file fragments = [] -with open(mergePlaceFile, 'rb') as fmp: +with open(mergePlaceFile, 'rt') as fmp: for line in fmp: m = line.split() name_and_rot = re.split(r'\*rotated|\*flipped', m[0]) # split the name and rotation/flip parts @@ -35,7 +35,7 @@ with open(mergePlaceFile, 'rb') as fmp: print ("* Starting merge of " + str(len(fragments)) + " board fragments...") -outf = gzip.open(fileOutName, 'wb') +outf = gzip.open(fileOutName, 'wt') outf.write("#VRML V2.0 utf8\n") pat_hdr = re.compile('^#VRML.*') @@ -54,7 +54,7 @@ for frag in fragments: fragId = str(fId).zfill(2) print ("* Adding " + frag["name"] + " (" + fileName + ") at (" + str(off_x_mm) + "," + str(off_y_mm) + "), rot=" + str(rot) + ", invZ=" + str(invertZ) + "...") - with open(fileName, 'rb') as f: + with open(fileName, 'rt') as f: for line in f: line = line.rstrip() # skip the headers (we write our own because there should be only 1 header) @@ -69,7 +69,7 @@ for frag in fragments: if pat_kicad_transform.match(line): print ("* Kicad VRML detected!") # todo: this is a 'hack' - z_offset = -board_thickness + z_offset = -(board_thickness / 2) # for upside-down modules, the offset needs to be reversed if (invertZ < 0): z_offset = -board_thickness - z_offset diff --git a/bin/create_board.sh b/bin/create_board.sh index 3ecc13c..7c29012 100755 --- a/bin/create_board.sh +++ b/bin/create_board.sh @@ -11,7 +11,7 @@ frame_rev="$3" bom_replace="$4" comp_img_offset="$5" -python_bin="python2.7" +python_bin="python3" ############################################################################################ diff --git a/bin/create_board_with_prefix.sh b/bin/create_board_with_prefix.sh index 212adcc..6e1326e 100755 --- a/bin/create_board_with_prefix.sh +++ b/bin/create_board_with_prefix.sh @@ -12,7 +12,7 @@ frame_rev="$4" bom_replace="$5" comp_img_offset="$6" -python_bin="python2.7" +python_bin="python3" ############################################################################################ diff --git a/bin/gen_iBOM.py b/bin/gen_iBOM.py index 49c7bc0..52a4921 100644 --- a/bin/gen_iBOM.py +++ b/bin/gen_iBOM.py @@ -24,7 +24,7 @@ sys.path.append("./bin/InteractiveHtmlBom/InteractiveHtmlBom/core") from lzstring import LZString if len(sys.argv) < 12: - print "Error! Please specify all the parameters!" + print ("Error! Please specify all the parameters!") sys.exit(1) boardName = sys.argv[1] @@ -45,7 +45,7 @@ inch_to_mm = 25.4 def getFormat(xI, xD, yI, yD, yInvert): - print "* Format: ", xI, xD, yI, yD + print ("* Format: ", xI, xD, yI, yD) fmt_pat_x = re.compile(r'^([0-9]{'+xI+'})([0-9]{'+xD+'})$') fmt_pat_y = re.compile(r'^([0-9]{'+yI+'})([0-9]{'+yD+'})$') return [fmt_pat_x, fmt_pat_y, yInvert] @@ -85,7 +85,7 @@ def readGerber(filePath, yInvert): maxCoord = [ -99999, -99999 ] format = getFormat("2", "5", "2", "5", yInvert) invertedMask = False - with open(filePath, 'rb') as f: + with open(filePath, 'rt') as f: for line in f: line = line.strip() #print line @@ -101,7 +101,7 @@ def readGerber(filePath, yInvert): apertNum = int(apertCircle.group(1)) apertSize = apertCircle.group(2) apertList[apertNum] = ["circle", apertSize] - # print "* Aperture C: " + str(apertNum) + " = " + apertSize + # print ("* Aperture C: " + str(apertNum) + " = " + apertSize) apertRect = apert_rect_pat.match(line) if apertRect: @@ -109,7 +109,7 @@ def readGerber(filePath, yInvert): apertSizeX = apertRect.group(2) apertSizeY = apertRect.group(4) apertList[apertNum] = ["rect", [apertSizeX, apertSizeY]] - # print "* Aperture R: " + str(apertNum) + " = " + apertSizeX + " " + apertSizeY + # print ("* Aperture R: " + str(apertNum) + " = " + apertSizeX + " " + apertSizeY) op = op_pat.match(line) if op: @@ -151,7 +151,7 @@ def readGerber(filePath, yInvert): a = int(op) cur_aper_type = apertList[a][0] cur_size = apertList[a][1] - # print "* Changing aperture: ", a + # print ("* Changing aperture: ", a) else: cur_x = x cur_y = y @@ -187,7 +187,7 @@ def readFootprint(fpname, footprintsPath, des): pat_pad = re.compile(r'^\s*\(pad\s+\"?([0-9A-Z]+)\"?\s+(\w+)\s+(\w+)\s+\(at\s+([+\-0-9e\.]+)\s+([+\-0-9e\.]+)\s*([+\-0-9\.]+)?\)\s+\(size\s+([+\-0-9\.]+)\s+([+\-0-9\.]+)\)(\s*\(drill\s+([+\-0-9\.]+)\))?\s+\(layer[s]?\s+\"?([^\)]+)\)(\s*\(roundrect_rratio\s+([+\-0-9\.]+)\))?') fpFileName = footprintsPath + "/" + fpname + ".kicad_mod" - print("* Reading " + fpFileName) + print ("* Reading " + fpFileName) if not os.path.isfile(fpFileName): print("* Error! Footprint NOT FOUND! Skipping " + des) return None @@ -262,13 +262,13 @@ def readFootprints(bomPath, cplPath, footprintsPath, yInvert): footprints = {} rotations = {} # read rotations csv (to undo weird JLC's angles which are not footprint-oriented) - with open(fixRotationsPath, 'rb') as f: + with open(fixRotationsPath, 'rt') as f: next(f) reader = csv.reader(f, delimiter=',') for row in reader: rotations[row[0]] = float(row[1]) # read BOM csv - with open(bomPath, 'rb') as f: + with open(bomPath, 'rt') as f: next(f) reader = csv.reader(f, delimiter=',') for row in reader: @@ -281,7 +281,7 @@ def readFootprints(bomPath, cplPath, footprintsPath, yInvert): for b in bb: bom[b] = { "fp": row[2], "idx": idx } # read CPL csv - with open(cplPath, 'rb') as f: + with open(cplPath, 'rt') as f: reader = csv.reader(f, delimiter=',') for row in reader: if row[0] in bom: @@ -340,7 +340,7 @@ def readFootprints(bomPath, cplPath, footprintsPath, yInvert): ################################################################### -with open(htmlFileName, 'rb') as f: +with open(htmlFileName, 'rt') as f: html = f.read() f.close() @@ -382,7 +382,7 @@ print("* Adding the pcb image...") with open(renderedPcbPath, mode='rb') as f: renderedPcb = f.read() html = html.replace('___PCBDPI___', renderedPcbDpi) - html = html.replace('___PCBIMAGE___', 'data:image/png;base64,' + base64.b64encode(renderedPcb)) + html = html.replace('___PCBIMAGE___', 'data:image/png;base64,' + base64.b64encode(renderedPcb).decode('ascii')) print("* Adding the BOM data...") jsonBase64 = LZString().compress_to_base64(jsonText) @@ -394,5 +394,5 @@ with open(iBomFilePath, "wt") as wf: wf.write(html) wf.close() -print "Done!" +print ("Done!") diff --git a/bin/gerbmerge b/bin/gerbmerge index 3404d39..f78db83 160000 --- a/bin/gerbmerge +++ b/bin/gerbmerge @@ -1 +1 @@ -Subproject commit 3404d39dd773b8f5c745df547bcd812336f1621d +Subproject commit f78db83293e9737d4ee4e0d5d377e908ba9aba91 diff --git a/bin/process_BOM.py b/bin/process_BOM.py index e8278f6..c3c75ba 100644 --- a/bin/process_BOM.py +++ b/bin/process_BOM.py @@ -11,7 +11,7 @@ include_pat = re.compile(r'#include\s+\"?([^\"]+)\"?$') def read_repl_file(csv_name, repl_base_path, replList): print ("Reading replacement list from the CSV file " + csv_name + "...") - with open(csv_name, 'rb') as f: + with open(csv_name, 'rt') as f: reader = csv.reader(f, delimiter=',') for row in reader: # skip empty lines @@ -43,7 +43,7 @@ print ("Opening BOM file " + fileName + "...") rows = OrderedDict() emptyId = 1 -with open(fileName, 'rb') as f: +with open(fileName, 'rt') as f: reader = csv.reader(f, delimiter=',') print ("Searching for duplicates...") for row in reader: @@ -75,7 +75,7 @@ read_repl_file(repl_csv, repl_base_path, replList) print ("Processing the board replacements...") for r in replList: reDesignator = r[0] - for rowName in rows: + for rowName in list(rows.keys()): row = rows[rowName] if reDesignator in row[1]: print ("* Removing " + reDesignator + " from the old row...") @@ -110,15 +110,15 @@ for rowName in rows: print ("Saving...") -with open (fileName, 'wb') as new_f: +with open (fileName, 'wt') as new_f: rowIdx = 0 for rowName in rows: #for idx,item in enumerate(rows[rowName]): # print idx , ": ", item if rowIdx == 0: - writer = csv.writer(new_f, quoting=csv.QUOTE_NONE, quotechar='"', delimiter=',', lineterminator='\n') + writer = csv.writer(new_f, quoting=csv.QUOTE_NONE, quotechar='"', escapechar='', delimiter=',', lineterminator='\n') elif rowIdx == 1: - writer = csv.writer(new_f, quoting=csv.QUOTE_ALL, quotechar='"', delimiter=',', lineterminator='\n') + writer = csv.writer(new_f, quoting=csv.QUOTE_ALL, quotechar='"', escapechar='', delimiter=',', lineterminator='\n') row = rows[rowName] # restore empty names if rowName[0] == '_': diff --git a/bin/process_board.py b/bin/process_board.py index 10d859e..4cfa675 100644 --- a/bin/process_board.py +++ b/bin/process_board.py @@ -89,7 +89,8 @@ def print_module(name, prefixPath, moduleName, fileName, isBoard, isBottom): "*" + bottom + "Soldermask=%(prefix)s.GBS", "*" + bottom + "Silkscreen=%(prefix)s.GBO", "*Keepout=%(prefix)s.GKO", - "Drills=%(prefix)s.DRL"]) + "Drills=%(prefix)s.DRL", + "drillspth=%(prefix)s.DRL"]) if ((os.path.isfile(prefix + ".G1") and os.path.isfile(prefix + ".G2")) or isBoard == 1): write_lines(file, [ "*InnerLayer2=%(prefix)s.G1", @@ -105,7 +106,7 @@ def print_module(name, prefixPath, moduleName, fileName, isBoard, isBottom): def append_cpl(src_fname, dst_fname, x, y, mrot, isBottom, suffix = ""): print ("* appending the CPL with offset (" + str(x) + "," + str(y) + ")...") - with open(src_fname, 'rb') as src_f, open(dst_fname, 'a') as dst_f: + with open(src_fname, 'rt') as src_f, open(dst_fname, 'a') as dst_f: reader = csv.reader(src_f, delimiter=',') i=0 # skip header @@ -153,7 +154,7 @@ def append_cpl(src_fname, dst_fname, x, y, mrot, isBottom, suffix = ""): def append_bom(src_fname, dst_fname, suffix = ""): print ("* appending the BOM...") - with open(src_fname, 'rb') as src_f, open(dst_fname, 'a') as dst_f: + with open(src_fname, 'rt') as src_f, open(dst_fname, 'a') as dst_f: reader = csv.reader(src_f, delimiter=',') i = 0 # skip header @@ -332,7 +333,7 @@ p = subprocess.Popen([sys.executable, "bin/gerbmerge/gerbmerge", board_cfg_path], stdin=subprocess.PIPE) # pass 'y' symbol to the subprocess as if a user pressed 'yes' -p.communicate(input='y\n')[0] +p.communicate(input=b'y\n')[0] check_returncode(p.returncode) print ("Post-processing BOM...") @@ -340,9 +341,9 @@ try: out = subprocess.check_output([sys.executable, "bin/process_BOM.py", board_bom, bom_replace_csv_path], stderr=subprocess.STDOUT) - print (out) -except subprocess.CalledProcessError, e: - print ("BOM processing error:\n" + e.output) + print (out.decode('ascii')) +except subprocess.CalledProcessError as e: + print ("BOM processing error:\n" + e.output.decode('ascii')) sys.exit(2) print ("Merging Schematics...") @@ -383,14 +384,14 @@ result = subprocess.call([sys.executable, "bin/create_3d_components.py", check_returncode(result) print ("Rendering a 3D-model of the board components...") -result = subprocess.call([node_bin, "bin/render_vrml/render_components.js", +result = subprocess.call([sys.executable, "bin/render_vrml/render_components.py", board_misc_path_name + "-3D.wrl.gz", board_img_components, imageDpi]) check_returncode(result) print ("Creating a composite board image...") -result = subprocess.call([node_bin, "bin/render_vrml/render_board.js", +result = subprocess.call([sys.executable, "bin/render_vrml/render_board.py", board_img_top, board_img_outline, board_img_components, diff --git a/bin/render_gerber.py b/bin/render_gerber.py index 8e483e4..bc6547a 100644 --- a/bin/render_gerber.py +++ b/bin/render_gerber.py @@ -14,7 +14,7 @@ from gerber.render import RenderSettings from gerber.render.cairo_backend import GerberCairoContext if len(sys.argv) < 3: - print "Error! Please specify the gerber path, image filename and board side." + print ("Error! Please specify the gerber path, image filename and board side.") sys.exit(1) gerberPath = sys.argv[1] imageFileName = sys.argv[2] @@ -145,20 +145,20 @@ curTheme = jlcTheme # choose layers if boardSide == 'top': - print "* Top Side:" + print ("* Top Side:") boardLayers = pcb.top_layers + pcb.drill_layers isOutline = False elif boardSide == 'bottom': - print "* Bottom Side:" + print ("* Bottom Side:") boardLayers = pcb.bottom_layers + pcb.drill_layers isOutline = False elif boardSide == 'outline': - print "* Board Outline:" + print ("* Board Outline:") boardLayers = [pcb.outline_layer] + pcb.drill_layers curTheme = outlineTheme isOutline = True else: - print "Error! Please specify the valid board side." + print ("Error! Please specify the valid board side.") sys.exit(2) # render diff --git a/bin/render_vrml/.gitignore b/bin/render_vrml/.gitignore deleted file mode 100644 index 3c3629e..0000000 --- a/bin/render_vrml/.gitignore +++ /dev/null @@ -1 +0,0 @@ -node_modules diff --git a/bin/render_vrml/moderngl_mesh.py b/bin/render_vrml/moderngl_mesh.py new file mode 100644 index 0000000..9838956 --- /dev/null +++ b/bin/render_vrml/moderngl_mesh.py @@ -0,0 +1,93 @@ +''' + ModernGL extension for storing mesh data + [andreika]: Modified to support vertex colors instead of texture coords +''' + +import logging +import re +import struct + +from pyrr import aabb + + +log = logging.getLogger('ModernGL.ext.obj') + +RE_COMMENT = re.compile(r'#[^\n]*\n', flags=re.M) +RE_VERT = re.compile(r'^v\s+(-?\d+(?:\.\d+)?(?:[Ee]-?\d+)?)\s+(-?\d+(?:\.\d+)?(?:[Ee]-?\d+)?)\s+(-?\d+(?:\.\d+)?(?:[Ee]-?\d+)?)$') +RE_COLOR = re.compile(r'^vc\s+(-?\d+(?:\.\d+)?(?:[Ee]-?\d+)?)\s+(-?\d+(?:\.\d+)?(?:[Ee]-?\d+)?)(?:\s+(-?\d+(?:\.\d+)?(?:[Ee]-?\d+)?))?$') +RE_NORM = re.compile(r'^vn\s+(-?\d+(?:\.\d+)?(?:[Ee]-?\d+)?)\s+(-?\d+(?:\.\d+)?(?:[Ee]-?\d+)?)\s+(-?\d+(?:\.\d+)?(?:[Ee]-?\d+)?)$') +RE_FACE = re.compile(r'^f\s+(\d+)(/(\d+)?(/(\d+))?)?\s+(\d+)(/(\d+)?(/(\d+))?)?\s+(\d+)(/(\d+)?(/(\d+))?)?$') + +PACKER = 'lambda vx, vy, vz, cx, cy, cz, nx, ny, nz: struct.pack("%df", %s)' + + +def default_packer(vx, vy, vz, cx, cy, cz, nx, ny, nz): + return struct.pack('9f', vx, vy, vz, cx, cy, cz, nx, ny, nz) + + +def int_or_none(x): + return None if x is None else int(x) + + +def safe_float(x): + return 0.0 if x is None else float(x) + + +class Mesh: + def __init__(self, vert, color, norm, face): + self.vert = vert + self.color = color + self.norm = norm + self.face = face + # we need AABB to zoom in the model + self.aabb = aabb.create_zeros() + + def pack(self, packer=default_packer) -> bytes: + ''' + Args: + packer (str or lambda): The vertex attributes to pack. + + Returns: + bytes: The packed vertex data. + + Examples: + + .. code-block:: python + + import ModernGL + from ModernGL.ext import obj + + model = obj.Obj.open('box.obj') + + # default packer + data = model.pack() + + # same as the default packer + data = model.pack('vx vy vz tx ty tz nx ny nz') + + # pack vertices + data = model.pack('vx vy vz') + + # pack vertices and texture coordinates (xy) + data = model.pack('vx vy vz tx ty') + + # pack vertices and normals + data = model.pack('vx vy vz nx ny nz') + + # pack vertices with padding + data = model.pack('vx vy vz 0.0') + ''' + + if isinstance(packer, str): + nodes = packer.split() + packer = eval(PACKER % (len(nodes), ', '.join(nodes))) + + result = bytearray() + + for v, t, n in self.face: + vx, vy, vz = self.vert[v - 1] + cx, cy, cz = self.color[t - 1] if t is not None else (0.0, 0.0, 0.0) + nx, ny, nz = self.norm[n - 1] if n is not None else (0.0, 0.0, 0.0) + result += packer(vx, vy, vz, cx, cy, cz, nx, ny, nz) + + return bytes(result) diff --git a/bin/render_vrml/package.json b/bin/render_vrml/package.json deleted file mode 100644 index 079aaa8..0000000 --- a/bin/render_vrml/package.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "dependencies": { - "fs": "0.0.1-security", - "pngjs": "^6.0.0", - "puppeteer": "^5.5.0", - "zlib": "^1.0.5" - } -} diff --git a/bin/render_vrml/render_board.js b/bin/render_vrml/render_board.js deleted file mode 100644 index 158d65e..0000000 --- a/bin/render_vrml/render_board.js +++ /dev/null @@ -1,79 +0,0 @@ -/* - ############################################################################################ - # Hellen-One: A board rendering server script. - # (c) andreika - ############################################################################################ - - Node.js is required to run this script: https://nodejs.org/en/download/ - Also please install PNGJS Node modules before running: - > npm install --prefix ./bin/render_vrml pngjs -*/ - -function getPixel(img, x, y) { - if (x < 0 || y < 0 || x >= img.width || y >= img.height) - return [0, 0, 0, 0]; - var idx = (img.width * y + x) << 2; - return [ img.data[idx], img.data[idx + 1], img.data[idx + 2], img.data[idx + 3] ]; -} - -function createBoardImg(pcbImg, outlineImg, compImg, compImgOffset) { - var boardImg = new PNG({ - width: Math.max(pcbImg.width, outlineImg.width, compImg.width), - height: Math.max(pcbImg.height, outlineImg.height, compImg.height) }); - var pcbOffY = (pcbImg.height < outlineImg.height) ? -(outlineImg.height - pcbImg.height) : 0; - // Blit the pcbImg using the outlineImg mask and add compImg. - // We cannot use PNG.bitblt() for that - for (var y = 0; y < boardImg.height; y++) { - for (var x = 0; x < boardImg.width; x++) { - var bPixel = getPixel(pcbImg, x, y + pcbOffY); - var cPixel = getPixel(compImg, x + compImgOffset[0], y + compImgOffset[1]); - var a = parseFloat(cPixel[3]) / 255.0; - var na = 1.0 - a; - var idx = (boardImg.width * y + x) << 2; - boardImg.data[idx + 0] = na * bPixel[0] + a * cPixel[0]; - boardImg.data[idx + 1] = na * bPixel[1] + a * cPixel[1]; - boardImg.data[idx + 2] = na * bPixel[2] + a * cPixel[2]; - boardImg.data[idx + 3] = na * getPixel(outlineImg, x, y)[0] + a * 255; - } - } - return boardImg; -} - -try { - var PNG = require("pngjs").PNG; - // built-in modules - var fs = require("fs"); - - var args = process.argv.slice(2); - if (args.length < 4) { - console.log("* Error! Please specify correct arguments to run this script!"); - process.exit(1) - } - // arguments - var pcbImgFile = args[0]; - var outlineImgFile = args[1]; - var compImgFile = args[2]; - var boardImgFile = args[3]; - var compImgOffset = args[4].split(",").map((x) => parseInt(x)); - - console.log("* Reading the pcb image..."); - var pcbImg = PNG.sync.read(fs.readFileSync(pcbImgFile)); - console.log("* Reading the components image with offset (" + compImgOffset[0] + "," + compImgOffset[1] + ")..."); - var compImg = PNG.sync.read(fs.readFileSync(compImgFile)); - console.log("* Reading the outline image..."); - var outlineImg = PNG.sync.read(fs.readFileSync(outlineImgFile)); - - console.log("* Creating the final board image..."); - var boardImg = createBoardImg(pcbImg, outlineImg, compImg, compImgOffset); - console.log("* Saving the board image..."); - fs.writeFileSync(boardImgFile, PNG.sync.write(boardImg, { colorType: 6 })); - - console.log("* Done! Exiting..."); - -} catch(e) { - if (e.message.indexOf("Cannot find module") !== -1) { - console.error("Error! `pngjs` library is not installed? Try running `npm install --prefix ./bin/render_vrml`."); - } - console.log(e); - process.exit(e.code); -} diff --git a/bin/render_vrml/render_board.py b/bin/render_vrml/render_board.py new file mode 100644 index 0000000..5556bd6 --- /dev/null +++ b/bin/render_vrml/render_board.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python +############################################################################################ +# Hellen-One: A board rendering server script. +# Python 3.5+ is required. +# Dependencies: Pillow +# (c) andreika +############################################################################################ + +import os, sys +from PIL import Image + +if (len(sys.argv) < 6): + print ("* Error! Please specify correct arguments to run this script!") + sys.exit(1) + +pcbImgFile = sys.argv[1] +outlineImgFile = sys.argv[2] +compImgFile = sys.argv[3] +boardImgFile = sys.argv[4] +compImgOffset = [int(n) for n in sys.argv[5].split(",")] + +class ImageObject: + width = 0 + height = 0 + data = [] + +def getPixel(img, x, y): + if (x < 0 or y < 0 or x >= img.width or y >= img.height): + return [0, 0, 0, 0] + return img.data[x, y] + +def createBoardImg(pcbImg, outlineImg, compImg, compImgOffset): + width = max([pcbImg.width, outlineImg.width, compImg.width]) + height = max([pcbImg.height, outlineImg.height, compImg.height]) + boardImg = Image.new('RGBA', (width, height)) + + pcbOffY = -(outlineImg.height - pcbImg.height) if (pcbImg.height < outlineImg.height) else 0 + # Blit the pcbImg using the outlineImg mask and add compImg. + # We cannot use PNG.bitblt() for that + for y in range(0, boardImg.height): + for x in range(0, boardImg.width): + bPixel = getPixel(pcbImg, x, y + pcbOffY) + cPixel = getPixel(compImg, x + compImgOffset[0], y + compImgOffset[1]) + a = float(cPixel[3]) / 255.0 + na = 1.0 - a + pr = int(na * bPixel[0] + a * cPixel[0]) + pg = int(na * bPixel[1] + a * cPixel[1]) + pb = int(na * bPixel[2] + a * cPixel[2]) + pa = int(na * getPixel(outlineImg, x, y)[0] + a * 255) + boardImg.putpixel((x, y), (pr, pg, pb, pa)) + return boardImg + +def loadImage(fileName): + pimg = Image.open(fileName).convert("RGBA") + img = ImageObject() + img.data = pimg.load() + img.width = pimg.size[0] + img.height = pimg.size[1] + return img + +print ("* Reading the pcb image...") +pcbImg = loadImage(pcbImgFile) +print ("* Reading the components image with offset (" + str(compImgOffset[0]) + "," + str(compImgOffset[1]) + ")...") +compImg = loadImage(compImgFile) +print ("* Reading the outline image...") +outlineImg = loadImage(outlineImgFile) + +print ("* Creating the final board image...") +boardImg = createBoardImg(pcbImg, outlineImg, compImg, compImgOffset) +print ("* Saving the board image...") +boardImg.save(boardImgFile) + +print ("* Done! Exiting...") diff --git a/bin/render_vrml/render_components.js b/bin/render_vrml/render_components.js deleted file mode 100644 index 37c3203..0000000 --- a/bin/render_vrml/render_components.js +++ /dev/null @@ -1,100 +0,0 @@ -/* - ############################################################################################ - # Hellen-One: A 3D-component VRML rendering server script. - # (c) andreika - ############################################################################################ - - Node.js is required to run this script: https://nodejs.org/en/download/ - Also please install Puppeteer and PNGJS Node modules before running: - > npm install --prefix ./bin/render_vrml puppeteer - > npm install --prefix ./bin/render_vrml pngjs -*/ - -try { - const puppeteer = require("puppeteer"); - var PNG = require("pngjs").PNG; - // built-in modules - var fs = require("fs"); - const zlib = require("zlib"); - - var args = process.argv.slice(2); - if (args.length < 3) { - console.log("* Error! Please specify correct arguments to run this script!"); - process.exit(1) - } - // arguments - var vrmlFile = args[0]; - var compImgFile = args[1]; - var dpi = parseFloat(args[2]); - - console.log("* Starting Puppeteer (" + vrmlFile + " dpi=" + dpi + ")..."); - - (async () => { - const browser = await puppeteer.launch({ - timeout: 60000, - slowMo: 500, - headless: true, - args: ['--unlimited-storage', '--full-memory-crash-report'] - }); - const page = await browser.newPage(); - - var contentHtml = fs.readFileSync("bin/render_vrml/render_vrml.html", "utf8"); - - console.log("* Reading the input file..."); - var vrmlData = fs.readFileSync(vrmlFile); - var vrmlDataType = "octet-stream"; - if (vrmlFile.endsWith('.gz')) { - // we unzip it later, on the frontend side - vrmlDataType = "x-gzip"; - } - var vrmlDataBase64 = vrmlData.toString("base64"); - - // injecting the data to the client script - contentHtml = contentHtml.replace("___SCREEN_DPI___", dpi); - contentHtml = contentHtml.replace("___VRML_DATA___", "data:application/" + vrmlDataType + ";base64," + vrmlDataBase64); - - page.on("console", message => { - message.args().forEach(async (arg) => { - const val = await arg.jsonValue() - if (JSON.stringify(val) !== JSON.stringify({})) - console.log(`* Script: ` + val) - else { - const { type, subtype, description } = arg._remoteObject; - console.log(`* Script: ${type} ${subtype}:\n ${description}`) - } - }) - }) - .on("pageerror", ({ message }) => console.log(`* LOG_ERROR: ` + message)) - .on("response", response => console.log(`* Loading: ` + `${response.status()} ${response.url()}`)) - .on("requestfailed", request => console.log(`* Loading Failed: ` + `${request.failure() ? request.failure().errorText : "?"} ${request.url()}`)); - - process.on('unhandledRejection', (reason, p) => { - console.error('Unhandled Rejection at: Promise', p, 'reason:', reason); - browser.close(); - }); - - await page.setDefaultNavigationTimeout(90000); // 90 seconds - - console.log("* Executing the script (content size is " + contentHtml.length + " bytes)..."); - await page.setContent(contentHtml, { waitUntil: 'domcontentloaded' }); - console.log("* Waiting for completion..."); - const watchDog = page.waitForFunction("document.done", {timeout: 180000}); // 180 seconds - await watchDog; - - var screenWidth = await page.evaluate(() => document.compImgWidth); - var screenHeight = await page.evaluate(() => document.compImgHeight); - console.log("* Saving the screenshot (" + screenWidth + "x" + screenHeight + ")"); - await page.setViewport({width: screenWidth, height: screenHeight, deviceScaleFactor: 1, }); - await page.screenshot({path: compImgFile, omitBackground: true}); - await page.close(); - await browser.close(); - console.log("* Done! Exiting..."); - })(); - -} catch(e) { - if (e.message.indexOf("Cannot find module") !== -1) { - console.error("Error! `Puppeteer` or `pngjs` library is not installed? Try running `npm install --prefix ./bin/render_vrml`."); - } - console.log(e); - process.exit(e.code); -} diff --git a/bin/render_vrml/render_components.py b/bin/render_vrml/render_components.py new file mode 100644 index 0000000..7b2a7a9 --- /dev/null +++ b/bin/render_vrml/render_components.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python +############################################################################################ +# Hellen-One: A 3D-component VRML rendering server script. +# Python 3.5+ is required. +# Dependencies: ModernGL, Pillow, Pyrr, NumPy, PyVRML97, Gzip +# (c) andreika +############################################################################################ + +import os, sys +import numpy as np +from pyrr import Matrix33,Matrix44,Vector3,Vector4,aabb +from vrml.vrml97 import parser,basenodes +from vrml import * +import moderngl as ModernGL +from moderngl_mesh import Mesh +from PIL import Image +import gzip + +if (len(sys.argv) < 3): + print ("Error! Please specify the VRML+image file names and DPI.") + sys.exit(1) +fileName = sys.argv[1] +outFileName = sys.argv[2] +dpi = float(sys.argv[3]) + +curLocation = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))) + + +# uses ModernGL (OpenGL-based renderer with HW acceleration) +def render(model, outFileName, dpi): + vertex_data = model.pack('vx vy vz cx cy cz nx ny nz') + + # Context creation + ctx = ModernGL.create_standalone_context() + + # Shaders + vertex_shader_source = open(os.path.join(curLocation, 'shader.vert')).read() + fragment_shader_source = open(os.path.join(curLocation, 'shader.frag')).read() + prog = ctx.program(vertex_shader = vertex_shader_source, fragment_shader = fragment_shader_source) + + # Matrices and Uniforms + perspective = Matrix44.orthogonal_projection(model.aabb[0][0], model.aabb[1][0], model.aabb[0][1], model.aabb[1][1], 0.1, 1000.0) # left,right,top,bottom,near,far + lookat = Matrix44.look_at( + (0,0,300), + (0,0,0), + (0.0, 1.0, 0.0), + ) + + mvp = perspective * lookat + + prog['LightDir'].value = (0.0, 0.0, -1.0) + prog['AmbientColor'].value = (1.0, 1.0, 1.0, 0.25) + prog['Mvp'].write(mvp.astype('f4').tobytes()) + + # Vertex Buffer and Vertex Array + vbo = ctx.buffer(vertex_data) + vao = ctx.simple_vertex_array(prog, vbo, * ['in_vert', 'in_color', 'in_norm']) + + (width, height) = ((model.aabb[1][0:2] - model.aabb[0][0:2]) * dpi / 25.4 + 0.5).astype(int) + print ("* Image size = " + str(width) + "x" + str(height)) + + # Framebuffer + fbo = ctx.framebuffer(ctx.renderbuffer((width, height)), ctx.depth_renderbuffer((width, height)),) + + # Rendering + fbo.use() + ctx.enable(ModernGL.DEPTH_TEST) + ctx.clear(0.0, 0.0, 0.0, 0.0) + vao.render() + + # Save the image using Pillow + print ("* Saving to " + str(outFileName) + "...") + data = fbo.read(components=4, alignment=1) + img = Image.frombytes('RGBA', fbo.size, data, 'raw', 'RGBA', 0, -1) + img.save(outFileName) + +def addFaces(model, faces, mat, tm): + offs = len(model.vert) + # create an inverse transform matrix for normals + tnm = Matrix33(tm) + tnm = tnm.inverse.T + # add vertices + for i in range(0, len(faces.coord.point)): + # position + x = float(faces.coord.point[i][0]) + y = float(faces.coord.point[i][1]) + z = float(faces.coord.point[i][2]) + v = Vector4([x, y, z, 1.0]) + tv = tm * v + # get normal (if present) + try: + (nx,ny,nz) = (faces.normal.vector[i][0:3]) + nv = Vector3([nx, ny, nz]) + except AttributeError: + nv = Vector3([0,0,-1]) # default normal + tnv = tnm * nv + tnv.normalize() + model.vert.append((tv.x, tv.y, tv.z)) + model.norm.append((tnv.x, tnv.y, tnv.z)) + model.color.append((mat.diffuseColor[0], mat.diffuseColor[1], mat.diffuseColor[2])) + model.aabb = aabb.add_points(model.aabb, [tv.xyz]) + # add indices + numFaces = int(len(faces.coordIndex) / 4) + for i in range(0, numFaces): + for ci in range(0, 3): + idx = faces.coordIndex[i * 4 + ci] + # indices are 1-based + ind = int(idx + offs) + 1 + model.face.append((ind, ind, ind)) + +def processChildren(model, ch, tm): + # iterate through all nodes + for i,node in enumerate(ch): + if (type(node) is basenodes.Group): + processChildren(model, node.children, tm) + continue + if (type(node) is basenodes.Transform): + transform = Matrix44(node.localMatrices().data[0].tolist()) if (node.localMatrices().data[0] is not None) else Matrix44.identity() + # apply transform + combinedTransform = tm * transform + processChildren(model, node.children, combinedTransform) + continue + if (type(node) is basenodes.Shape and type(node.geometry) is basenodes.IndexedFaceSet): + addFaces(model, node.geometry, node.appearance.material, tm) + +############################################################################################ + +print ("* Reading " + fileName + "...") +if fileName.endswith('.gz'): + print ("* Decompressing GZip...") + data = gzip.open(fileName, "rt").read() +else: + data = open(fileName).read() + +print ("* Parsing the VRML data...") +parser = vrml97.parser.buildParser() +success, tags, next = parser.parse(data) +if (not success): + print("VRML Parsing error: parsed %s characters of %s"%(next, len(data))) + sys.exit(1) + +print ("* Processing the data...") +model = Mesh([], [], [], []) +processChildren(model, tags[1].children, Matrix44.identity()) + +print ("* Num.Vertices = " + str(len(model.vert)) + " Num.Indices = " + str(len(model.face))) +print ("* AABB = " + str(model.aabb)) + +print ("* Rendering...") +render(model, outFileName, dpi) + +print ("* Done!") diff --git a/bin/render_vrml/render_vrml.html b/bin/render_vrml/render_vrml.html deleted file mode 100644 index 0c7a784..0000000 --- a/bin/render_vrml/render_vrml.html +++ /dev/null @@ -1,106 +0,0 @@ - - diff --git a/bin/render_vrml/shader.frag b/bin/render_vrml/shader.frag new file mode 100644 index 0000000..f40d846 --- /dev/null +++ b/bin/render_vrml/shader.frag @@ -0,0 +1,32 @@ +#version 330 + +uniform vec4 AmbientColor; +uniform vec3 LightDir; + +in vec3 v_vert; +in vec3 v_norm; +in vec3 v_color; + +out vec4 f_color; + +void main() { + // clip negative fragments (below the board surface level) + if (v_vert.z < 0.1) { + discard; + return; + } + // we use abs() to make normals compatible with both CW and CCW faces + vec3 n = normalize(abs(v_norm)); + // calc luminosity + float lum = dot(n, LightDir); + lum = acos(lum) / 3.14159265; + lum = clamp(lum, 0.0, 1.0); + lum = lum * lum; + lum = smoothstep(0.0, 1.0, lum); + + // modulate + vec3 lcolor = v_color * lum; + // add ambient color + vec3 color = lcolor * (1.0 - AmbientColor.a) + AmbientColor.rgb * AmbientColor.a; + f_color = vec4(color, 1.0); +} diff --git a/bin/render_vrml/shader.vert b/bin/render_vrml/shader.vert new file mode 100644 index 0000000..50b95af --- /dev/null +++ b/bin/render_vrml/shader.vert @@ -0,0 +1,18 @@ +#version 330 + +uniform mat4 Mvp; + +in vec3 in_vert; +in vec3 in_norm; +in vec3 in_color; + +out vec3 v_vert; +out vec3 v_norm; +out vec3 v_color; + +void main() { + v_vert = in_vert; + v_norm = in_norm; + v_color = in_color; + gl_Position = Mvp * vec4(v_vert, 1.0); +}