From a3ca646df12d48613df1420ec8cd0779d5c55cb8 Mon Sep 17 00:00:00 2001 From: Tim 'mithro' Ansell Date: Mon, 1 Mar 2021 18:52:54 -0800 Subject: [PATCH 01/12] Revert accidental commit to version.py Signed-off-by: Tim 'mithro' Ansell --- fasm/version.py | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/fasm/version.py b/fasm/version.py index 0406adc4..a67184c0 100644 --- a/fasm/version.py +++ b/fasm/version.py @@ -13,22 +13,17 @@ # This file is auto-generated by the update_version.py script. # ** WARNING ** -version_str = "0.0.2.post66" -version_tuple = (0, 0, 2, 66) -try: - from packaging.version import Version as V - pversion = V("0.0.2.post66") -except ImportError: - pass +version_str = "UNKNOWN" +version_tuple = (0, 0, 0, 0) -git_hash = "c0b734e6d373fcffd0522acdd102814bcefb626a" -git_describe = "v0.0.2-66-gc0b734e" +git_hash = "UNKNOWN" +git_describe = "UNKNOWN" git_msg = """\ -commit c0b734e6d373fcffd0522acdd102814bcefb626a -Author: Tim 'mithro' Ansell -Date: Fri Feb 19 13:05:20 2021 -0800 +commit UNKNOWN +Author: UNKNOWN +Date: UNKNOWN - Improve the warning message when falling back to the textX. + UNKNOWN Signed-off-by: Tim 'mithro' Ansell From ece6d9e6d9aa6491cab11c9c09ea838bcf9a82dd Mon Sep 17 00:00:00 2001 From: Tim 'mithro' Ansell Date: Mon, 1 Mar 2021 19:08:13 -0800 Subject: [PATCH 02/12] Remove fasm/version.py file. Signed-off-by: Tim 'mithro' Ansell --- fasm/version.py | 30 ------------------------------ setup.py | 8 ++------ 2 files changed, 2 insertions(+), 36 deletions(-) delete mode 100644 fasm/version.py diff --git a/fasm/version.py b/fasm/version.py deleted file mode 100644 index a67184c0..00000000 --- a/fasm/version.py +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -# Copyright (C) 2017-2020 The SymbiFlow Authors. -# -# Use of this source code is governed by a ISC-style -# license that can be found in the LICENSE file or at -# https://opensource.org/licenses/ISC -# -# SPDX-License-Identifier: ISC - -# ** WARNING ** -# This file is auto-generated by the update_version.py script. -# ** WARNING ** - -version_str = "UNKNOWN" -version_tuple = (0, 0, 0, 0) - -git_hash = "UNKNOWN" -git_describe = "UNKNOWN" -git_msg = """\ -commit UNKNOWN -Author: UNKNOWN -Date: UNKNOWN - - UNKNOWN - - Signed-off-by: Tim 'mithro' Ansell - -""" diff --git a/setup.py b/setup.py index 784b8ad4..e71d42f3 100644 --- a/setup.py +++ b/setup.py @@ -33,12 +33,8 @@ # Read in the version information FASM_VERSION_FILE = os.path.join(__dir__, 'fasm', 'version.py') -with open(FASM_VERSION_FILE) as f: - if 'UNKNOWN' in f.read(): - print( - "Running update_version.py to generate {}".format( - FASM_VERSION_FILE)) - subprocess.check_call(['python', 'update_version.py'], cwd=__dir__) +if not os.path.exists(FASM_VERSION_FILE): + subprocess.check_call(['python', 'update_version.py'], cwd=__dir__) with open(FASM_VERSION_FILE) as f: lines = f.readlines() version_line = [v.strip() for v in lines if v.startswith('version_str')] From 720a2aa3136e37a643c069f22549957f90cf2203 Mon Sep 17 00:00:00 2001 From: Tim 'mithro' Ansell Date: Sun, 7 Mar 2021 23:34:21 -0800 Subject: [PATCH 03/12] Try harder to use the same Python as running setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e71d42f3..520ec89b 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ # Read in the version information FASM_VERSION_FILE = os.path.join(__dir__, 'fasm', 'version.py') if not os.path.exists(FASM_VERSION_FILE): - subprocess.check_call(['python', 'update_version.py'], cwd=__dir__) + subprocess.check_call([sys.executable, 'update_version.py'], cwd=__dir__) with open(FASM_VERSION_FILE) as f: lines = f.readlines() version_line = [v.strip() for v in lines if v.startswith('version_str')] From f6b981aa9e3f9a55af321077dd3374e226366aa2 Mon Sep 17 00:00:00 2001 From: Tim 'mithro' Ansell Date: Mon, 1 Mar 2021 20:51:29 -0800 Subject: [PATCH 04/12] Set file as cython with language_level=3. Signed-off-by: Tim 'mithro' Ansell --- fasm/parser/antlr_to_tuple.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fasm/parser/antlr_to_tuple.pyx b/fasm/parser/antlr_to_tuple.pyx index c57c6053..16bba284 100644 --- a/fasm/parser/antlr_to_tuple.pyx +++ b/fasm/parser/antlr_to_tuple.pyx @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +# cython: language_level=3 # -*- coding: utf-8 -*- # # Copyright (C) 2020 The SymbiFlow Authors. From bb27dae82ec2c32b43b840d86e2a6d86bb24f0f6 Mon Sep 17 00:00:00 2001 From: Tim 'mithro' Ansell Date: Thu, 18 Feb 2021 11:01:58 -0800 Subject: [PATCH 05/12] Improve setup.py, cleanup requirement and remove setup.cfg file. `setup_requires` needs to be specified in *both* `setup.py` and `pyproject.toml`. The `requirements.txt` file should use `-e .` to install the module (and thus get the requirements). If a local or non-PyPi version of a module should be used to satisfy a requirement for either `setup_requires` or `install_requires` then it should also be specified in the `requirements.txt` file. Using `setup.py` is much clearer than `setup.cfg` and it looks like the file is going to get replaced by `pyproject.toml` in the future, see https://github.com/pypa/setuptools/issues/1688 The generated Cython C file is included in the source distribution, so only people building from the git repository need to have Cython installed (fixing import issue). Signed-off-by: Tim 'mithro' Ansell --- MANIFEST.in | 11 +++++++---- Makefile | 6 +++++- pyproject.toml | 7 ++++++- requirements.txt | 6 +++--- setup.cfg | 5 ----- setup.py | 36 ++++++++++++++++++++++++++++++------ 6 files changed, 51 insertions(+), 20 deletions(-) delete mode 100644 setup.cfg diff --git a/MANIFEST.in b/MANIFEST.in index 42e8397b..272ac0b2 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -33,17 +33,20 @@ prune docs/env # textx based parser recursive-include fasm/parser *.tx -recursive-include fasm/parser *.pyx -exclude fasm/parser/*.c # antlr based parser include src/* include src/antlr/* +# cython extension to antlr based parser +recursive-include fasm/parser *.pyx # Excludes -global-exclude .git exclude .gitmodules exclude update_version.py prune third_party prune .github -prune __pycache__ +prune */.git +prune */__pycache__ +global-exclude *.py[cod] +global-exclude *.o +global-exclude *.so diff --git a/Makefile b/Makefile index a5116601..6068011f 100644 --- a/Makefile +++ b/Makefile @@ -35,7 +35,11 @@ setup.py: fasm/version.py # Build/install into the conda environment. # ------------------------------------------------------------------------ build-clean: - rm -rf dist fasm.egg-info + rm -rf build dist fasm.egg-info + rm -f fasm/parser/antlr_to_tuple.c + rm -f fasm/parser/*.so + +clean:: build-clean .PHONY: build-clean diff --git a/pyproject.toml b/pyproject.toml index e9f294dd..3c0a31af 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,2 +1,7 @@ [build-system] -requires = ["setuptools", "wheel", "cython"] +requires = [ + "cython", + "setuptools>=42", + "wheel", +] +build-backend = "setuptools.build_meta" diff --git a/requirements.txt b/requirements.txt index e3a9915f..1b4631e1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,9 @@ +# Get fasm tooling requirements +-e . +# Testing check-manifest -cython flake8 pytest -textx tox twine -wheel yapf==0.24.0 diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 33aea034..00000000 --- a/setup.cfg +++ /dev/null @@ -1,5 +0,0 @@ -[metadata] -license_files = LICENSE - -[bdist_wheel] -universal=1 diff --git a/setup.py b/setup.py index 520ec89b..b26a2683 100644 --- a/setup.py +++ b/setup.py @@ -17,13 +17,13 @@ import sys import traceback -from Cython.Build import cythonize from distutils.command.build import build from distutils.version import LooseVersion from setuptools import Extension from setuptools.command.build_ext import build_ext from setuptools.command.develop import develop from setuptools.command.install import install +from setuptools.command.sdist import sdist __dir__ = os.path.dirname(os.path.abspath(__file__)) @@ -248,7 +248,15 @@ def run(self): super().run() +class SdistCommand(sdist): + def run(self): + from Cython.Build import cythonize + cythonize("fasm/parser/antlr_to_tuple.pyx") + super().run() + + setuptools.setup( + # Package human readable information name="fasm", version=version, author="SymbiFlow Authors", @@ -257,24 +265,40 @@ def run(self): long_description=long_description, long_description_content_type="text/markdown", url="https://github.com/SymbiFlow/fasm", - packages=setuptools.find_packages(exclude=('tests*', )), - install_requires=['textx'], - include_package_data=True, + license="ISC", + license_files=["LICENSE"], classifiers=[ "Programming Language :: Python :: 3", "License :: OSI Approved :: ISC License (ISCL)", "Operating System :: OS Independent", ], + # Package contents control + packages=setuptools.find_packages(exclude=['tests*']), + include_package_data=True, entry_points={ 'console_scripts': ['fasm=fasm.tool:main'], }, + # Requirements + python_requires=">=3.6", + setup_requires=[ # WARNING: Must be kept in sync with pyproject.toml + "cython", + "setuptools>=42", + "wheel", + ], + install_requires=[ + 'textx', + ], + # C extension building ext_modules=[ - CMakeExtension('parse_fasm', sourcedir='src', prefix='fasm/parser') - ] + cythonize("fasm/parser/antlr_to_tuple.pyx"), + CMakeExtension('parse_fasm', sourcedir='src', prefix='fasm/parser'), + Extension( + "fasm.parser.antlr_to_tuple", ['fasm/parser/antlr_to_tuple.c']), + ], cmdclass={ 'build_ext': AntlrCMakeBuild, 'build': BuildCommand, 'develop': DevelopCommand, 'install': InstallCommand, + 'sdist': SdistCommand, }, ) From 0487b5474b43b1138e14b26c964efdffdefef4d0 Mon Sep 17 00:00:00 2001 From: Tim 'mithro' Ansell Date: Mon, 1 Mar 2021 21:11:00 -0800 Subject: [PATCH 06/12] Rework the extensions in setup.py Signed-off-by: Tim 'mithro' Ansell --- pyproject.toml | 2 ++ requirements.txt | 4 ++++ setup.py | 57 ++++++++++++++++++++++++++++++++++++++---------- 3 files changed, 51 insertions(+), 12 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3c0a31af..89738e7e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,4 +1,6 @@ [build-system] +# WARNING: This needs to be kept in sync with the setup_requires in setup.py +# WARNING: This needs to be kept in sync with requirements.txt requires = [ "cython", "setuptools>=42", diff --git a/requirements.txt b/requirements.txt index 1b4631e1..b0780b84 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,9 @@ # Get fasm tooling requirements -e . +# WARNING: Must be kept in sync with pyproject.toml +cython +setuptools>=42 +wheel # Testing check-manifest flake8 diff --git a/setup.py b/setup.py index b26a2683..cbbefa04 100644 --- a/setup.py +++ b/setup.py @@ -45,8 +45,16 @@ version = version_value[1:-1] +# C extensions +extensions = [] +cmdclass = {} + +# Antlr based parser +# ------------------------------------------------------------------------ # Based on: https://www.benjack.io/2018/02/02/python-cpp-revisited.html # GitHub: https://github.com/benjaminjack/python_cpp_example + + class CMakeExtension(Extension): def __init__(self, name, sourcedir='', prefix=''): Extension.__init__(self, name, sources=[]) @@ -248,6 +256,20 @@ def run(self): super().run() +extensions += [ + CMakeExtension('parse_fasm', sourcedir='src', prefix='fasm/parser'), +] +cmdclass['build_ext'] = AntlrCMakeBuild +cmdclass['build'] = BuildCommand +cmdclass['develop'] = DevelopCommand +cmdclass['install'] = InstallCommand +# ------------------------------------------------------------------------ + +# Cython based accelerator +# ------------------------------------------------------------------------ +# Cython recommends shipping the .c file as part of your sdist package. + + class SdistCommand(sdist): def run(self): from Cython.Build import cythonize @@ -255,6 +277,27 @@ def run(self): super().run() +CYTHON_EXT_FILEBASE = 'fasm/parser/antlr_to_tuple' +if os.path.exists(CYTHON_EXT_FILEBASE + '.c'): + # Building from sdist which already includes the generated + # `antlr_to_tuple.c` file, so can treat it like any other C extensions. + extensions += [ + Extension( + "fasm.parser.antlr_to_tuple", [CYTHON_EXT_FILEBASE + '.c']), + ] +else: + # Building without a `antlr_to_tuple.c` file, so need to use Cython to + # generate the new `antlr_to_tuple.c` file from the `antlr_to_tuple.pyx`. + from Cython.Build import cythonize + extensions += [ + Extension( + "fasm.parser.antlr_to_tuple", [CYTHON_EXT_FILEBASE + '.pyx']), + ] + extensions = cythonize(extensions) + cmdclass['sdist'] = SdistCommand +# ------------------------------------------------------------------------ + + setuptools.setup( # Package human readable information name="fasm", @@ -289,16 +332,6 @@ def run(self): 'textx', ], # C extension building - ext_modules=[ - CMakeExtension('parse_fasm', sourcedir='src', prefix='fasm/parser'), - Extension( - "fasm.parser.antlr_to_tuple", ['fasm/parser/antlr_to_tuple.c']), - ], - cmdclass={ - 'build_ext': AntlrCMakeBuild, - 'build': BuildCommand, - 'develop': DevelopCommand, - 'install': InstallCommand, - 'sdist': SdistCommand, - }, + ext_modules=extensions, + cmdclass=cmdclass, ) From 780be59a057a34485f4c75217e765b31dc246b15 Mon Sep 17 00:00:00 2001 From: Tim 'mithro' Ansell Date: Mon, 1 Mar 2021 19:42:46 -0800 Subject: [PATCH 07/12] Create alias for `make test` Signed-off-by: Tim 'mithro' Ansell --- Makefile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Makefile b/Makefile index 6068011f..b9ed7321 100644 --- a/Makefile +++ b/Makefile @@ -83,6 +83,11 @@ test: fasm/version.py | $(CONDA_ENV_PYTHON) .PHONY: test +tests: test + @true + +.PHONY: tests + # Find files to apply tools to while ignoring files. define with_files $(IN_CONDA_ENV) git ls-files | grep -ve '^third_party\|^\.|^env' | grep -e $(1) | xargs -r -P $$(nproc) $(2) From f14d8b6aa6fa35607239e87722507ebd89285766 Mon Sep 17 00:00:00 2001 From: Tim 'mithro' Ansell Date: Sun, 21 Feb 2021 09:43:38 -0800 Subject: [PATCH 08/12] Remove Python 3.5 from wheel workflow. Signed-off-by: Tim 'mithro' Ansell --- .github/workflows/wheel.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 6392bd62..f11f331d 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: [3.5, 3.6, 3.7, 3.8, 3.9] + python-version: [3.6, 3.7, 3.8, 3.9] fail-fast: false steps: - uses: actions/checkout@v2 From 32178da8b4d18a07d329df16d375577dac58549d Mon Sep 17 00:00:00 2001 From: Tim 'mithro' Ansell Date: Sun, 21 Feb 2021 10:45:35 -0800 Subject: [PATCH 09/12] Cleanup of the workflow files. Steps are; 1) Get Python + PIP. 2) Run `update_version.py` 3) Install package's system dependencies. 4) Install package. Signed-off-by: Tim 'mithro' Ansell --- .github/workflows/check-install.yml | 29 +++++------- .github/workflows/publish-to-pypi.yml | 66 +++++++++++++++++---------- .github/workflows/wheel.yml | 26 +++++++---- 3 files changed, 72 insertions(+), 49 deletions(-) diff --git a/.github/workflows/check-install.yml b/.github/workflows/check-install.yml index 8793a2b1..481778f2 100644 --- a/.github/workflows/check-install.yml +++ b/.github/workflows/check-install.yml @@ -17,32 +17,31 @@ jobs: runs-on: ${{ matrix.os }} steps: - - name: Install dependencies (Ubuntu) + - name: 🐍 Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.x + + - name: Install pip + run: | + pip install --upgrade pip + + - name: Install package's system dependencies (Ubuntu) if: startsWith(matrix.os, 'ubuntu') run: | sudo apt-get update sudo apt-get install -y cmake default-jre-headless uuid-dev libantlr4-runtime-dev - - name: Install dependencies (Mac OS X) + - name: Install package's system dependencies (Mac OS X) if: startsWith(matrix.os, 'macos') run: | true - - name: Install dependencies (Windows) + - name: Install package's system dependencies (Windows) if: startsWith(matrix.os, 'windows') run: | true - - name: 🐍 Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.x - - - name: Install pip+wheel - run: | - pip install --upgrade wheel - pip install --upgrade pip - - name: Test installation shell: bash run: | @@ -51,7 +50,3 @@ jobs: - name: Run Smoke Test run: | fasm --help - -# PyPi: -# runs-on: ubuntu-20.04 -# name: PyPi diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml index 375f6dc4..7d7f57cb 100644 --- a/.github/workflows/publish-to-pypi.yml +++ b/.github/workflows/publish-to-pypi.yml @@ -19,25 +19,28 @@ jobs: fetch-depth: 0 submodules: true - - name: Install dependencies (Ubuntu) - run: | - sudo apt-get update - sudo apt-get install -y cmake default-jre-headless uuid-dev libantlr4-runtime-dev - - name: 🐍 Set up Python uses: actions/setup-python@v2 with: python-version: 3.x - - name: Install build dependencies + - name: Install packaging tooling run: | pip install -U pip - pip install -r requirements.txt + pip install twine + + - name: Update version + run: | python update_version.py - - name: Install package dependencies + - name: Install package's system dependencies (Ubuntu) + run: | + sudo apt-get update + sudo apt-get install -y cmake default-jre-headless uuid-dev libantlr4-runtime-dev + + - name: Install package run: | - python setup.py install + pip install -e . - name: 🚧 Build distribution 📦 run: | @@ -87,12 +90,24 @@ jobs: - name: 🐍 Set up Python uses: actions/setup-python@v2 - - name: Install build dependencies + - name: Install packaging tooling run: | pip install -U pip pip install twine auditwheel + + - name: Update version + run: | python update_version.py + # The "Install package's system dependencies" and "Install package" steps + # are /actually/ included in the `python-wheels-manylinux-build` action. + - name: Install package's system dependencies + run: | + true + - name: Install package + run: | + true + - name: 🚧 Build distribution 📦 uses: RalfG/python-wheels-manylinux-build@v0.3.3-manylinux2010_x86_64 with: @@ -164,30 +179,33 @@ jobs: fetch-depth: 0 submodules: true - - name: Install dependencies (Mac OS X) - if: startsWith(matrix.os, 'macos') - run: | - true - - - name: Install dependencies (Windows) - if: startsWith(matrix.os, 'windows') - run: | - true - - name: 🐍 Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - name: Install build dependencies + - name: Install packaging tooling run: | pip install -U pip - pip install -r requirements.txt + pip install twine auditwheel + + - name: Update version + run: | python update_version.py - - name: Install package dependencies + - name: Install package's system dependencies (Mac OS X) + if: startsWith(matrix.os, 'macos') + run: | + true + + - name: Install package's system dependencies (Windows) + if: startsWith(matrix.os, 'windows') + run: | + true + + - name: Install package run: | - python setup.py install + pip install -e . - name: 🚧 Build distribution 📦 run: | diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index f11f331d..ce201578 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -16,24 +16,34 @@ jobs: # Always clone the full depth so git-describe works. fetch-depth: 0 submodules: true - - name: Install dependencies - run: | - sudo apt update - sudo apt install cmake default-jre-headless uuid-dev libantlr4-runtime-dev + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: python -m pip install --upgrade -r requirements.txt + + - name: Install package's system dependencies + run: | + sudo apt update + sudo apt install cmake default-jre-headless uuid-dev libantlr4-runtime-dev + - name: Generate version - run: python update_version.py + run: | + python update_version.py + + - name: Install dependencies + run: | + python -m pip install --upgrade -r requirements.txt + - name: Build wheels - run: python setup.py bdist_wheel + run: | + python setup.py bdist_wheel + - name: Test wheel installation run: | python -m pip install dist/*.whl (cd tests; python test_simple.py) + - uses: actions/upload-artifact@v2 with: name: fasm From 830eb223f471d5c00ba95a6ef67ee11d0bcc69bc Mon Sep 17 00:00:00 2001 From: Tim 'mithro' Ansell Date: Sun, 21 Feb 2021 12:00:53 -0800 Subject: [PATCH 10/12] Rework the Github Actions workflows. * Create a composite action for setting up the system. * Expand `check-install.yml` to check that installing works from all of; - GitHub - Checked out locally with `python setup.py install` - Checked out locally with `pip install -e .` - Building and then installing wheels. * Remove `wheel.yml` as covered by `check-install.yml`'s building and then installing wheels. Signed-off-by: Tim 'mithro' Ansell --- .github/workflows/check-functionality.yml | 36 +++++++ .github/workflows/check-install.yml | 105 ++++++++++++++++----- .github/workflows/check-style.yml | 31 ++++++ .github/workflows/checkout/action.yaml | 13 +++ .github/workflows/presubmit.yml | 72 -------------- .github/workflows/publish-to-pypi.yml | 98 +++---------------- .github/workflows/system-setup/action.yaml | 85 +++++++++++++++++ .github/workflows/wheel.yml | 50 ---------- 8 files changed, 262 insertions(+), 228 deletions(-) create mode 100644 .github/workflows/check-functionality.yml create mode 100644 .github/workflows/check-style.yml create mode 100644 .github/workflows/checkout/action.yaml delete mode 100644 .github/workflows/presubmit.yml create mode 100644 .github/workflows/system-setup/action.yaml delete mode 100644 .github/workflows/wheel.yml diff --git a/.github/workflows/check-functionality.yml b/.github/workflows/check-functionality.yml new file mode 100644 index 00000000..ef7c3e6a --- /dev/null +++ b/.github/workflows/check-functionality.yml @@ -0,0 +1,36 @@ +name: Functionality + +on: + push: + pull_request: + +jobs: + + Functionality: + runs-on: ubuntu-latest + + strategy: + matrix: + antlr_runtime_type: [static, shared] + include: + - { python-version: 3.5, TOXENV: py35 } + - { python-version: 3.6, TOXENV: py36 } + - { python-version: 3.7, TOXENV: py37 } + - { python-version: 3.8, TOXENV: py38 } + - { python-version: 3.9, TOXENV: py39 } + fail-fast: false + + name: Functionality on Python ${{ matrix.python-version }} (with ${{ matrix.antlr_runtime_type}} antlr) + + steps: + - run: | + echo "${{ github.repository }}/.github/workflows/system-setup@${{ github.sha }}" + + - uses: mithro/fasm/.github/workflows/system-setup@setupcfg + with: + development-tools: true + python-version: ${{ matrix.python-version }} + + - name: Run Tests + run: | + ANTLR4_RUNTIME_TYPE=${{ matrix.antlr_runtime_type }} tox -e ${{ matrix.TOXENV }} diff --git a/.github/workflows/check-install.yml b/.github/workflows/check-install.yml index 481778f2..9fa52e42 100644 --- a/.github/workflows/check-install.yml +++ b/.github/workflows/check-install.yml @@ -10,43 +10,106 @@ jobs: GitHub: strategy: matrix: - os: [windows-latest, macos-latest, ubuntu-20.04] + os: [windows-latest, macos-latest, ubuntu-latest] fail-fast: false - name: GitHub runs-on: ${{ matrix.os }} steps: - - name: 🐍 Set up Python - uses: actions/setup-python@v2 + - uses: '${{ github.repository }}/.github/workflows/system-setup@${{ github.sha }}' with: - python-version: 3.x + os: ${{ matrix.os }} + git-checkout: false - - name: Install pip + - name: Test installation + shell: bash run: | - pip install --upgrade pip + pip install git+https://github.com/${GITHUB_REPOSITORY}.git@${GITHUB_SHA}#egg=fasm - - name: Install package's system dependencies (Ubuntu) - if: startsWith(matrix.os, 'ubuntu') + - name: Run smoke test run: | - sudo apt-get update - sudo apt-get install -y cmake default-jre-headless uuid-dev libantlr4-runtime-dev + fasm --help + +# - name: Run tests against installed version +# run: | +# cd tests; python test_simple.py + + Checkout: + strategy: + matrix: + os: [windows-latest, macos-latest, ubuntu-latest] + cmd: + - python setup.py install + - pip install . + - pip install -e . # Editable install + fail-fast: false + + name: Checkout with '${{ matrix.cmd }}' + runs-on: ${{ matrix.os }} - - name: Install package's system dependencies (Mac OS X) - if: startsWith(matrix.os, 'macos') + steps: + - uses: '${{ github.repository }}/.github/workflows/system-setup@${{ github.sha }}' + with: + os: ${{ matrix.os }} + + - name: Install using '${{ matrix.cmd }}' run: | - true + ${{ matrix.cmd }} - - name: Install package's system dependencies (Windows) - if: startsWith(matrix.os, 'windows') + - name: Run smoke test run: | - true + fasm --help - - name: Test installation - shell: bash + - name: Run tests against installed version run: | - pip install git+https://github.com/${GITHUB_REPOSITORY}.git@${GITHUB_SHA}#egg=fasm + cd tests; python test_simple.py + + Wheel: + strategy: + matrix: + os: [windows-latest, macos-latest, ubuntu-latest] + fail-fast: false - - name: Run Smoke Test + runs-on: ${{ matrix.os }} + + steps: + - uses: '${{ github.repository }}/.github/workflows/system-setup@${{ github.sha }}' + with: + os: ${{ matrix.os }} + + - name: Build wheel + run: | + python setup.py bdist_wheel + + - name: Upload wheel + uses: actions/upload-artifact@v2 + with: + name: fasm + path: dist + + - name: Install wheel + run: | + pip install dist/*.whl + + - name: Run smoke test run: | fasm --help + + - name: Run tests against installed version + run: | + cd tests; python test_simple.py + + make-env: + strategy: + matrix: + os: [windows-latest, macos-latest, ubuntu-latest] + + name: make-env (Conda) + runs-on: ${{ matrix.os }} + + steps: + - uses: '${{ github.repository }}/.github/workflows/checkout@${{ github.sha }}' + + - name: Run tests + run: | + make tests diff --git a/.github/workflows/check-style.yml b/.github/workflows/check-style.yml new file mode 100644 index 00000000..4b3cc8c7 --- /dev/null +++ b/.github/workflows/check-style.yml @@ -0,0 +1,31 @@ +name: Style + +on: + push: + pull_request: + +jobs: + + Style: + runs-on: ubuntu-latest + + steps: + - uses: '${{ github.repository }}/.github/workflows/system-setup@${{ github.sha }}' + with: + development-tools: true + + - name: Check license headers + run: make check-license + + - name: Python style check + run: | + make format lint + test $(git status --porcelain | wc -l) -eq 0 || { git diff; false; } + + - name: Python script checks + run: make check-python-scripts + + - name: C++ style check + run: | + make format-cpp + test $(git status --porcelain | wc -l) -eq 0 || { git diff; false; } diff --git a/.github/workflows/checkout/action.yaml b/.github/workflows/checkout/action.yaml new file mode 100644 index 00000000..f3f575d3 --- /dev/null +++ b/.github/workflows/checkout/action.yaml @@ -0,0 +1,13 @@ +name: 🧰 Checkout +description: "Checkout the git repository correctly" + +runs: + using: "composite" + + steps: + - name: 🧰 Checkout + uses: actions/checkout@v2 + with: + # Always clone the full depth so git-describe works. + fetch-depth: 0 + submodules: true diff --git a/.github/workflows/presubmit.yml b/.github/workflows/presubmit.yml deleted file mode 100644 index aca2f95f..00000000 --- a/.github/workflows/presubmit.yml +++ /dev/null @@ -1,72 +0,0 @@ -# Checks code that code meets requirements for a pull request. -# Any automated checks for code quality and compliance belongs here. -name: presubmit -on: [push, pull_request] -jobs: - check: - name: Source checks - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v2 - with: - # Always clone the full depth so git-describe works. - fetch-depth: 0 - submodules: true - - name: Set up Python - uses: actions/setup-python@v2 - - name: Install dependencies - run: | - sudo apt update - sudo apt install clang-format - python -m pip install --upgrade pip - pip install -r requirements.txt - - name: Python style check - run: | - make format lint - test $(git status --porcelain | wc -l) -eq 0 || { git diff; false; } - - name: C++ style check - run: | - make format-cpp - test $(git status --porcelain | wc -l) -eq 0 || { git diff; false; } - - name: Check license - run: make check-license - - name: Python checks - run: make check-python-scripts - test: - name: Test Python package - runs-on: ubuntu-20.04 - strategy: - matrix: - antlr_runtime_type: [static, shared] - python_version: [3.5, 3.6, 3.7, 3.8, 3.9] - include: - - python-version: 3.5 - TOXENV: py35 - - python-version: 3.6 - TOXENV: py36 - - python-version: 3.7 - TOXENV: py37 - - python-version: 3.8 - TOXENV: py38 - - python-version: 3.9 - TOXENV: py39 - fail-fast: false - steps: - - uses: actions/checkout@v2 - with: - # Always clone the full depth so git-describe works. - fetch-depth: 0 - submodules: true - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - sudo apt update - sudo apt install cmake default-jre-headless uuid-dev libantlr4-runtime-dev - python -m pip install --upgrade pip - pip install -r requirements.txt - python update_version.py - - name: Tox - run: ANTLR4_RUNTIME_TYPE=${{ matrix.antlr_runtime_type }} tox -e ${{ matrix.TOXENV }} diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml index 7d7f57cb..2b0c8500 100644 --- a/.github/workflows/publish-to-pypi.yml +++ b/.github/workflows/publish-to-pypi.yml @@ -8,35 +8,12 @@ on: jobs: Source: - runs-on: ubuntu-20.04 name: Source steps: - - name: 🧰 Checkout - uses: actions/checkout@v2 + - uses: '${{ github.repository }}/.github/workflows/system-setup@${{ github.sha }}' with: - # Always clone the full depth so git-describe works. - fetch-depth: 0 - submodules: true - - - name: 🐍 Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.x - - - name: Install packaging tooling - run: | - pip install -U pip - pip install twine - - - name: Update version - run: | - python update_version.py - - - name: Install package's system dependencies (Ubuntu) - run: | - sudo apt-get update - sudo apt-get install -y cmake default-jre-headless uuid-dev libantlr4-runtime-dev + packaging-tools: true - name: Install package run: | @@ -70,50 +47,26 @@ jobs: strategy: matrix: include: - - { name: '3.6', python-version: 'cp36-cp36m' } - - { name: '3.7', python-version: 'cp37-cp37m' } - - { name: '3.8', python-version: 'cp38-cp38' } - - { name: '3.9', python-version: 'cp39-cp39' } + - { python-version: '3.6', manylinux-python: 'cp36-cp36m' } + - { python-version: '3.7', manylinux-python: 'cp37-cp37m' } + - { python-version: '3.8', manylinux-python: 'cp38-cp38' } + - { python-version: '3.9', manylinux-python: 'cp39-cp39' } fail-fast: false name: '${{ matrix.name }} • manylinux' - runs-on: ubuntu-latest steps: - - name: 🧰 Checkout - uses: actions/checkout@v2 + - uses: '${{ github.repository }}/.github/workflows/system-setup@${{ github.sha }}' with: - # Always clone the full depth so git-describe works. - fetch-depth: 0 - submodules: true - - - name: 🐍 Set up Python - uses: actions/setup-python@v2 - - - name: Install packaging tooling - run: | - pip install -U pip - pip install twine auditwheel - - - name: Update version - run: | - python update_version.py - - # The "Install package's system dependencies" and "Install package" steps - # are /actually/ included in the `python-wheels-manylinux-build` action. - - name: Install package's system dependencies - run: | - true - - name: Install package - run: | - true + python-version: ${{ matrix.python-version }} + packaging-tools: true - name: 🚧 Build distribution 📦 uses: RalfG/python-wheels-manylinux-build@v0.3.3-manylinux2010_x86_64 with: build-requirements: 'cython' pre-build-command: 'bash .github/workflows/manylinux-install-cmake.sh' - python-versions: ${{ matrix.python-version }} + python-versions: ${{ matrix.manylinux-python }} #pip-wheel-args: '-w ./dist --no-deps --verbose' - name: List distribution 📦 @@ -172,36 +125,11 @@ jobs: runs-on: ${{ matrix.os }} steps: - - name: 🧰 Checkout - uses: actions/checkout@v2 - with: - # Always clone the full depth so git-describe works. - fetch-depth: 0 - submodules: true - - - name: 🐍 Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + - uses: '${{ github.repository }}/.github/workflows/system-setup@${{ github.sha }}' with: + os: ${{ matrix.os }} python-version: ${{ matrix.python-version }} - - - name: Install packaging tooling - run: | - pip install -U pip - pip install twine auditwheel - - - name: Update version - run: | - python update_version.py - - - name: Install package's system dependencies (Mac OS X) - if: startsWith(matrix.os, 'macos') - run: | - true - - - name: Install package's system dependencies (Windows) - if: startsWith(matrix.os, 'windows') - run: | - true + packaging-tools: true - name: Install package run: | diff --git a/.github/workflows/system-setup/action.yaml b/.github/workflows/system-setup/action.yaml new file mode 100644 index 00000000..4b3461ec --- /dev/null +++ b/.github/workflows/system-setup/action.yaml @@ -0,0 +1,85 @@ +name: "Setup system for package" +description: "Set up system with Python environment and dependencies ready for the package." +inputs: + python-version: + description: 'Python version to use.' + required: true + default: 3.x + os: + description: 'Operating system in use.' + required: true + default: ubuntu-latest + system-dependencies: + description: 'Install the system dependencies on the operating system.' + required: true + default: true + packaging-tools: + description: 'Install the tools required for packaging.' + required: false + default: false + git-checkout: + description: 'Download the repository from git.' + required: true + default: true + development-tools: + description: 'Install the tools required for development.' + required: true + default: false + +runs: + using: "composite" + + steps: + - name: 🐍 Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ input.python-version }} + + - name: Install latest pip + run: | + pip install -U pip + + - name: Install package's system dependencies (Ubuntu) + if: inputs.system-dependencies && startsWith(inputs.os, 'ubuntu') + run: | + sudo apt-get update + sudo apt-get install -y cmake default-jre-headless uuid-dev libantlr4-runtime-dev + + - name: Install package's system dependencies (Mac OS X) + if: inputs.system-dependencies && startsWith(inputs.os, 'macos') + run: | + true + + - name: Install package's system dependencies (Windows) + if: inputs.system-dependencies && startsWith(inputs.os, 'windows') + run: | + true + + - name: Install packaging tooling + if: inputs.packaging-tools + run: | + pip install twine auditwheel + + - uses: ./.github/workflows/checkout + if: inputs.git-checkout + + - name: Install development system dependencies (Ubuntu) + if: inputs.development-tools && startsWith(inputs.os, 'ubuntu') + run: | + sudo apt-get update + sudo apt-get install -y clang-format + + - name: Install development system dependencies (Mac OS X) + if: inputs.development-tools && startsWith(inputs.os, 'macos') + run: | + true + + - name: Install development system dependencies (Windows) + if: inputs.development-tools && startsWith(inputs.os, 'windows') + run: | + true + + - name: Install development tooling + if: inputs.development-tools + run: | + pip install -r requirements.txt diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml deleted file mode 100644 index ce201578..00000000 --- a/.github/workflows/wheel.yml +++ /dev/null @@ -1,50 +0,0 @@ -# Builds a binary distibutable package. -# Minimal functionality checks may be run as part of the build process, but -# source code checks and extensive functionality checks (e.g. tox) belong in presubmit.yml -name: Python wheels -on: [push, pull_request] -jobs: - wheels: - runs-on: ubuntu-20.04 - strategy: - matrix: - python-version: [3.6, 3.7, 3.8, 3.9] - fail-fast: false - steps: - - uses: actions/checkout@v2 - with: - # Always clone the full depth so git-describe works. - fetch-depth: 0 - submodules: true - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Install package's system dependencies - run: | - sudo apt update - sudo apt install cmake default-jre-headless uuid-dev libantlr4-runtime-dev - - - name: Generate version - run: | - python update_version.py - - - name: Install dependencies - run: | - python -m pip install --upgrade -r requirements.txt - - - name: Build wheels - run: | - python setup.py bdist_wheel - - - name: Test wheel installation - run: | - python -m pip install dist/*.whl - (cd tests; python test_simple.py) - - - uses: actions/upload-artifact@v2 - with: - name: fasm - path: dist From ce32b75888cac9e9835cd5826dceefe05dad2014 Mon Sep 17 00:00:00 2001 From: Tim 'mithro' Ansell Date: Wed, 24 Feb 2021 15:07:11 -0800 Subject: [PATCH 11/12] Rework the GitHub Actions files. - Use the new mithro/actions-includes for common functionality. - Refactor common functionality into their own include actions. Signed-off-by: Tim 'mithro' Ansell --- .../checkout/action.yaml | 10 +- .../download-and-run-tests/action.yaml | 46 ++ .../download-and-run-tests/fasm-version.py | 26 + .../download-and-run-tests/get-tests.sh | 54 ++ .../system-setup/action.yaml | 24 +- .github/actions/upload-to-pypi/action.yaml | 60 ++ .github/workflows-src/Makefile | 46 ++ .github/workflows-src/check-functionality.yml | 40 + .github/workflows-src/check-install.yml | 199 +++++ .github/workflows-src/check-style.yml | 39 + .github/workflows-src/publish-to-pypi.yml | 115 +++ .github/workflows-src/requirements.txt | 1 + .github/workflows/check-functionality.yml | 79 +- .github/workflows/check-install.yml | 771 ++++++++++++++++-- .github/workflows/check-style.yml | 56 +- .github/workflows/publish-to-pypi.yml | 215 +++-- Makefile | 18 + requirements.txt | 2 + 18 files changed, 1636 insertions(+), 165 deletions(-) rename .github/{workflows => actions}/checkout/action.yaml (51%) create mode 100644 .github/actions/download-and-run-tests/action.yaml create mode 100644 .github/actions/download-and-run-tests/fasm-version.py create mode 100644 .github/actions/download-and-run-tests/get-tests.sh rename .github/{workflows => actions}/system-setup/action.yaml (77%) create mode 100644 .github/actions/upload-to-pypi/action.yaml create mode 100644 .github/workflows-src/Makefile create mode 100644 .github/workflows-src/check-functionality.yml create mode 100644 .github/workflows-src/check-install.yml create mode 100644 .github/workflows-src/check-style.yml create mode 100644 .github/workflows-src/publish-to-pypi.yml create mode 100644 .github/workflows-src/requirements.txt diff --git a/.github/workflows/checkout/action.yaml b/.github/actions/checkout/action.yaml similarity index 51% rename from .github/workflows/checkout/action.yaml rename to .github/actions/checkout/action.yaml index f3f575d3..7da7fe73 100644 --- a/.github/workflows/checkout/action.yaml +++ b/.github/actions/checkout/action.yaml @@ -1,8 +1,16 @@ +# Copyright (C) 2021 The SymbiFlow Authors. +# +# Use of this source code is governed by a ISC-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/ISC +# +# SPDX-License-Identifier: ISC + name: 🧰 Checkout description: "Checkout the git repository correctly" runs: - using: "composite" + using: "includes" steps: - name: 🧰 Checkout diff --git a/.github/actions/download-and-run-tests/action.yaml b/.github/actions/download-and-run-tests/action.yaml new file mode 100644 index 00000000..b4eaceac --- /dev/null +++ b/.github/actions/download-and-run-tests/action.yaml @@ -0,0 +1,46 @@ +# Copyright (C) 2021 The SymbiFlow Authors. +# +# Use of this source code is governed by a ISC-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/ISC +# +# SPDX-License-Identifier: ISC + +name: Download and run tests +description: "Download the tests from GitHub and run them." + +runs: + using: "includes" + + steps: + + - name: Smoke Test - Run fasm tool + run: | + fasm --help + + - name: Smoke Test - Import fasm module + shell: python + run: | + import fasm + + - name: Smoke Test - Print fasm version info + includes-script: fasm-version.py + + - name: Getting the tests + includes-script: get-tests.sh + + - name: List Tests + shell: bash + run: | + echo "::group::Top directory" + ls -l tests + echo "::endgroup::" + echo "::group::Files found" + find tests -type f | sort + echo "::endgroup::" + + - name: Run Tests + shell: bash + run: | + cd tests + python test_simple.py diff --git a/.github/actions/download-and-run-tests/fasm-version.py b/.github/actions/download-and-run-tests/fasm-version.py new file mode 100644 index 00000000..dc0abd3c --- /dev/null +++ b/.github/actions/download-and-run-tests/fasm-version.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python + +import fasm.version + +l = [] + +print() +print(' FASM library version info') +print('='*75) + +kl = max(len(k) for k in dir(fasm.version)) +for k in dir(fasm.version): + if '__' in k: + continue + v = getattr(fasm.version, k) + if isinstance(v, str) and '\n' in v: + l.append((k,v)) + else: + print(" {!s}: {!r}".format(k.rjust(kl), v)) + +for k, v in l: + print() + print(k) + print('-'*75) + print(v) + print('-'*75) diff --git a/.github/actions/download-and-run-tests/get-tests.sh b/.github/actions/download-and-run-tests/get-tests.sh new file mode 100644 index 00000000..683ee272 --- /dev/null +++ b/.github/actions/download-and-run-tests/get-tests.sh @@ -0,0 +1,54 @@ +if [ -d tests ]; then + echo "::group::Using existing tests" + ls -l tests + echo "::endgroup::" +else + echo "::group::Event info" + cat ${GITHUB_EVENT_PATH} + echo "::endgroup::" + echo "::group::GitHub info" + echo "GITHUB_REPOSITORY: ${GITHUB_REPOSITORY}" + echo " GITHUB_ACTOR: ${GITHUB_ACTOR}" + echo " GITHUB_REF: ${GITHUB_REF}" + echo " GITHUB_BASE_REF: ${GITHUB_BASE_REF}" + echo " GITHUB_HEAD_REF: ${GITHUB_HEAD_REF}" + echo " GITHUB_SHA: ${GITHUB_SHA}" + echo "::endgroup::" + echo "::group::Downloading tests from ${GITHUB_REPOSITORY}" + set -x + mkdir .checkout-tests + cd .checkout-tests + git init + git config core.sparseCheckout true + if [ -f .git/info/sparse-checkout ]; then + rm .git/info/sparse-checkout + fi + echo "tests/*" >> .git/info/sparse-checkout + echo "examples/*" >> .git/info/sparse-checkout + git remote add origin ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git + git fetch --all + git remote -v + if [ ! -z "${GITHUB_REF}" ]; then + git fetch --refmap='' origin ${GITHUB_REF}:refs/remotes/origin/merge || true + fi + if [ ! -z "${GITHUB_BASE_REF}" ]; then + git fetch --refmap='' origin refs/heads/${GITHUB_BASE_REF}:refs/remotes/origin/base || true + fi + if [ ! -z "${GITHUB_HEAD_REF}" ]; then + git fetch --refmap='' origin refs/heads/${GITHUB_HEAD_REF}:refs/remotes/origin/head || true + fi + git remote show origin + git branch -v -a + + git show-ref ${GITHUB_SHA} || true + git rev-parse --verify "sha^${GITHUB_SHA}" || true + + git fetch -q https://github.com/SymbiFlow/fasm.git ${GITHUB_SHA} + git rev-parse FETCH_HEAD + git checkout ${GITHUB_SHA} + for i in *; do + cp -rvf $i .. + done + cd .. + echo "::endgroup::" +fi diff --git a/.github/workflows/system-setup/action.yaml b/.github/actions/system-setup/action.yaml similarity index 77% rename from .github/workflows/system-setup/action.yaml rename to .github/actions/system-setup/action.yaml index 4b3461ec..f832dff0 100644 --- a/.github/workflows/system-setup/action.yaml +++ b/.github/actions/system-setup/action.yaml @@ -1,3 +1,11 @@ +# Copyright (C) 2021 The SymbiFlow Authors. +# +# Use of this source code is governed by a ISC-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/ISC +# +# SPDX-License-Identifier: ISC + name: "Setup system for package" description: "Set up system with Python environment and dependencies ready for the package." inputs: @@ -27,13 +35,13 @@ inputs: default: false runs: - using: "composite" + using: "includes" steps: - name: 🐍 Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: - python-version: ${{ input.python-version }} + python-version: ${{ inputs.python-version }} - name: Install latest pip run: | @@ -58,28 +66,28 @@ runs: - name: Install packaging tooling if: inputs.packaging-tools run: | - pip install twine auditwheel + pip install twine auditwheel build - - uses: ./.github/workflows/checkout + - includes: /checkout if: inputs.git-checkout - - name: Install development system dependencies (Ubuntu) + - name: Install developer tooling's system dependencies (Ubuntu) if: inputs.development-tools && startsWith(inputs.os, 'ubuntu') run: | sudo apt-get update sudo apt-get install -y clang-format - - name: Install development system dependencies (Mac OS X) + - name: Install developer tooling's system dependencies (Mac OS X) if: inputs.development-tools && startsWith(inputs.os, 'macos') run: | true - - name: Install development system dependencies (Windows) + - name: Install developer tooling's system dependencies (Windows) if: inputs.development-tools && startsWith(inputs.os, 'windows') run: | true - - name: Install development tooling + - name: Install development tools if: inputs.development-tools run: | pip install -r requirements.txt diff --git a/.github/actions/upload-to-pypi/action.yaml b/.github/actions/upload-to-pypi/action.yaml new file mode 100644 index 00000000..f4736267 --- /dev/null +++ b/.github/actions/upload-to-pypi/action.yaml @@ -0,0 +1,60 @@ +# Copyright (C) 2021 The SymbiFlow Authors. +# +# Use of this source code is governed by a ISC-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/ISC +# +# SPDX-License-Identifier: ISC + +name: "Publish packages into PyPI" +description: "Check the packages and then publish packages onto Test and real versions." +inputs: + type: + description: 'Type of packages to publish to PyPi.' + required: true + root_repo: + description: 'Repository name that should be publishing packages to PyPi.' + required: true + root_branch: + description: 'Default branch to publish packages from.' + required: true + default: refs/heads/master + +runs: + using: "includes" + + steps: + - name: ✔︎ Check 📦 + run: | + for WHEEL in dist/*.whl; do + echo + echo "::group::Checking $WHEEL" + echo + python -m zipfile --list $WHEEL + echo + auditwheel show $WHEEL + echo + twine check $WHEEL + echo + echo "::endgroup::" + done + + - name: 📤 Publish ${{ inputs.type }} to Test PyPI + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_TEST_PASSWORD }} + if: env.TWINE_PASSWORD != null + run: | + twine upload --skip-existing --verbose --repository testpypi dist/* + + - name: 📤 Publish source to PyPI + if: | + (github.ref == inputs.root_branch) && + (github.event_name != 'pull_request') && + (github.repository == inputs.root_repo) && + env.TWINE_PASSWORD != null + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + twine upload dist/* diff --git a/.github/workflows-src/Makefile b/.github/workflows-src/Makefile new file mode 100644 index 00000000..9e02ad59 --- /dev/null +++ b/.github/workflows-src/Makefile @@ -0,0 +1,46 @@ +# Copyright (C) 2017-2021 The SymbiFlow Authors. +# +# Use of this source code is governed by a ISC-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/ISC +# +# SPDX-License-Identifier: ISC + +# Set up a Python environment to run the actions_include tool on. +ENV_DIR = venv +PYTHON = $(ENV_DIR)/bin/python3 +ACTIVATE = source $(ENV_DIR)/bin/activate; + +env: requirements.txt + rm -rf $(ENV_DIR) + virtualenv --copies $(ENV_DIR) + $(ACTIVATE) pip install -r $< + touch --reference=$< $(PYTHON) + +.PHONY: env + +$(PYTHON): requirements.txt + make env + +# Generate the output files +SRC_YAML = $(wildcard *.yml) +OUT_YAML = $(addprefix ../workflows/,$(SRC_YAML)) + +../workflows/%.yml: %.yml | $(PYTHON) + @echo + @echo Updating $@ + @echo ------------------------------------------------ + $(ACTIVATE) python -m actions_includes $< $@ + @echo ------------------------------------------------ + +update: + @for F in $(SRC_YAML); do touch $$F; done + make build + +build: $(OUT_YAML) | $(PYTHON) + @true + +info: + @echo 'Output files: $(OUT_YAML)' + +.PHONY: info diff --git a/.github/workflows-src/check-functionality.yml b/.github/workflows-src/check-functionality.yml new file mode 100644 index 00000000..67e602b7 --- /dev/null +++ b/.github/workflows-src/check-functionality.yml @@ -0,0 +1,40 @@ +# Copyright (C) 2017-2021 The SymbiFlow Authors. +# +# Use of this source code is governed by a ISC-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/ISC +# +# SPDX-License-Identifier: ISC + +on: + push: + pull_request: + +name: Functionality +jobs: + + functionality: + runs-on: ubuntu-20.04 + + strategy: + matrix: + antlr_runtime_type: [static, shared] + include: + - { python-version: 3.5, TOXENV: py35 } + - { python-version: 3.6, TOXENV: py36 } + - { python-version: 3.7, TOXENV: py37 } + - { python-version: 3.8, TOXENV: py38 } + - { python-version: 3.9, TOXENV: py39 } + fail-fast: false + + name: Functionality on Python ${{ matrix.python-version }} (with ${{ matrix.antlr_runtime_type}} antlr) + + steps: + - includes: /system-setup + with: + development-tools: true + python-version: ${{ matrix.python-version }} + + - name: Run Tests + run: | + ANTLR4_RUNTIME_TYPE=${{ matrix.antlr_runtime_type }} tox -e ${{ matrix.TOXENV }} diff --git a/.github/workflows-src/check-install.yml b/.github/workflows-src/check-install.yml new file mode 100644 index 00000000..ff629970 --- /dev/null +++ b/.github/workflows-src/check-install.yml @@ -0,0 +1,199 @@ +# Copyright (C) 2017-2021 The SymbiFlow Authors. +# +# Use of this source code is governed by a ISC-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/ISC +# +# SPDX-License-Identifier: ISC + +name: Install from + +on: + push: + pull_request: + + +jobs: + + # Install directly using pip from GitHub + # ---------------------------------------------------------------------- + GitHub: + strategy: + matrix: + os: [windows-latest, macos-latest, ubuntu-20.04] + fail-fast: false + + runs-on: ${{ matrix.os }} + + steps: + - includes: /system-setup + with: + os: ${{ matrix.os }} + git-checkout: false + + - name: Installing directly from GitHub + shell: bash + run: | + pip install --verbose git+https://github.com/${GITHUB_REPOSITORY}.git@${GITHUB_SHA}#egg=fasm + + - includes: /download-and-run-tests + # ---------------------------------------------------------------------- + + # Install using a local checkout + # ---------------------------------------------------------------------- + Checkout: + strategy: + matrix: + os: [windows-latest, macos-latest, ubuntu-20.04] + cmd: + - python setup.py install + - pip install --verbose . + - pip install --verbose -e . # Editable install + fail-fast: false + + name: Checkout with '${{ matrix.cmd }}' (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + + steps: + - includes: /system-setup + with: + os: ${{ matrix.os }} + + - name: Installing using '${{ matrix.cmd }}' + run: | + ${{ matrix.cmd }} + + - includes: /download-and-run-tests + # ---------------------------------------------------------------------- + + # Install into self-contained `make-env` environment. + # ---------------------------------------------------------------------- + make-env: + strategy: + matrix: + os: [windows-latest, macos-latest, ubuntu-20.04] + fail-fast: false + + name: make-env (Conda) + runs-on: ${{ matrix.os }} + + steps: + - includes: /system-setup + with: + os: ${{ matrix.os }} + + - name: Run tests + run: | + make test + # ---------------------------------------------------------------------- + + # Install from a sdist package + # ---------------------------------------------------------------------- + BuildSdist: + strategy: + matrix: + os: [ubuntu-20.04] + fail-fast: false + + runs-on: ${{ matrix.os }} + + steps: + - includes: /system-setup + with: + os: ${{ matrix.os }} + packaging-tools: true + + - name: Build wheel + run: | + python -m build --sdist + + - name: Upload wheel + uses: actions/upload-artifact@v2 + with: + name: fasm-sdist + path: dist + + sdist: + strategy: + matrix: + os: [windows-latest, macos-latest, ubuntu-20.04] + fail-fast: false + + runs-on: ${{ matrix.os }} + needs: BuildSdist + + steps: + - includes: /system-setup + with: + os: ${{ matrix.os }} + git-checkout: false + + - name: Download sdist + uses: actions/download-artifact@v2 + with: + name: fasm-sdist + path: dist + + - name: Installing using the sdist + shell: bash + run: | + ls -l dist/* + pip install --verbose dist/* + + - includes: /download-and-run-tests + # ---------------------------------------------------------------------- + + # Install from a binary wheel package + # ---------------------------------------------------------------------- + BuildWheel: + strategy: + matrix: + os: [windows-latest, macos-latest, ubuntu-20.04] + fail-fast: false + + runs-on: ${{ matrix.os }} + + steps: + - includes: /system-setup + with: + os: ${{ matrix.os }} + packaging-tools: true + + - name: Build wheel + run: | + python -m build --wheel + + - name: Upload wheel + uses: actions/upload-artifact@v2 + with: + name: fasm-wheel-${{ matrix.os }} + path: dist + + Wheel: + strategy: + matrix: + os: [windows-latest, macos-latest, ubuntu-20.04] + fail-fast: false + + runs-on: ${{ matrix.os }} + needs: BuildWheel + + steps: + - includes: /system-setup + with: + os: ${{ matrix.os }} + git-checkout: false + + - name: Download wheel + uses: actions/download-artifact@v2 + with: + name: fasm-wheel-${{ matrix.os }} + path: dist + + - name: Installing using the wheel + run: | + ls -l dist/* + pip install --verbose dist/*.whl + + - includes: /download-and-run-tests + # ---------------------------------------------------------------------- diff --git a/.github/workflows-src/check-style.yml b/.github/workflows-src/check-style.yml new file mode 100644 index 00000000..f9ab4f7e --- /dev/null +++ b/.github/workflows-src/check-style.yml @@ -0,0 +1,39 @@ +# Copyright (C) 2021 The SymbiFlow Authors. +# +# Use of this source code is governed by a ISC-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/ISC +# +# SPDX-License-Identifier: ISC + +name: Style + +on: + push: + pull_request: + +jobs: + + Style: + runs-on: ubuntu-20.04 + + steps: + - includes: /system-setup + with: + development-tools: true + + - name: Check license headers + run: make check-license + + - name: Python style check + run: | + make format lint + test $(git status --porcelain | wc -l) -eq 0 || { git diff; false; } + + - name: Python script checks + run: make check-python-scripts + + - name: C++ style check + run: | + make format-cpp + test $(git status --porcelain | wc -l) -eq 0 || { git diff; false; } diff --git a/.github/workflows-src/publish-to-pypi.yml b/.github/workflows-src/publish-to-pypi.yml new file mode 100644 index 00000000..2a1e4eae --- /dev/null +++ b/.github/workflows-src/publish-to-pypi.yml @@ -0,0 +1,115 @@ +# Copyright (C) 2017-2021 The SymbiFlow Authors. +# +# Use of this source code is governed by a ISC-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/ISC +# +# SPDX-License-Identifier: ISC + +name: PyPI + +on: + push: + pull_request: + workflow_dispatch: + +jobs: + + Source: + name: Source + runs-on: ubuntu-20.04 + + steps: + - includes: /system-setup + with: + packaging-tools: true + + - name: Install package + run: | + pip install -e . + + - name: 🚧 Build distribution 📦 + run: | + python -m build --sdist . + + - includes: /upload-to-pypi + with: + root_repo: SymbiFlow/fasm + type: source + + Linux: + strategy: + matrix: + include: + - { python-version: '3.6', manylinux-python: 'cp36-cp36m' } + - { python-version: '3.7', manylinux-python: 'cp37-cp37m' } + - { python-version: '3.8', manylinux-python: 'cp38-cp38' } + - { python-version: '3.9', manylinux-python: 'cp39-cp39' } + fail-fast: false + + name: '${{ matrix.python-version }} • manylinux' + runs-on: ubuntu-latest + + steps: + - includes: /system-setup + with: + python-version: ${{ matrix.python-version }} + system-dependencies: false + packaging-tools: true + + - name: 🚧 Build distribution 📦 + uses: RalfG/python-wheels-manylinux-build@v0.3.3-manylinux2010_x86_64 + with: + build-requirements: 'cython' + pre-build-command: 'bash .github/workflows/manylinux-install-cmake.sh' + python-versions: ${{ matrix.manylinux-python }} + #pip-wheel-args: '-w ./dist --no-deps --verbose' + + - name: List distribution 📦 + run: | + # Fix permissions + echo "::group::Fixing permission" + ls -l dist/* + echo $USER + whoami + sudo chown -R $USER dist + ls -l dist/* + echo "::endgroup::" + # Remove the non-manylinux versions + rm -v dist/*linux_x86_64*.whl + ls -l dist/* + + - includes: /upload-to-pypi + with: + root_repo: SymbiFlow/fasm + type: wheels + + MacAndWindows: + strategy: + matrix: + os: [windows-latest, macos-latest] + python-version: [ '3.6', '3.7', '3.8', '3.9', 'pypy-3.6', 'pypy-3.7' ] + fail-fast: false + + name: ${{ matrix.python-version }} • ${{ matrix.os }} + runs-on: ${{ matrix.os }} + + steps: + - includes: /system-setup + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + packaging-tools: true + + - name: Install package + run: | + pip install -e . + + - name: 🚧 Build distribution 📦 + run: | + python setup.py bdist_wheel + + - includes: /upload-to-pypi + with: + root_repo: SymbiFlow/fasm + type: wheels diff --git a/.github/workflows-src/requirements.txt b/.github/workflows-src/requirements.txt new file mode 100644 index 00000000..b64c895d --- /dev/null +++ b/.github/workflows-src/requirements.txt @@ -0,0 +1 @@ +git+https://github.com/mithro/actions-includes.git#egg=actions-includes diff --git a/.github/workflows/check-functionality.yml b/.github/workflows/check-functionality.yml index ef7c3e6a..67975918 100644 --- a/.github/workflows/check-functionality.yml +++ b/.github/workflows/check-functionality.yml @@ -1,36 +1,71 @@ -name: Functionality +# Copyright (C) 2017-2021 The SymbiFlow Authors. +# +# Use of this source code is governed by a ISC-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/ISC +# +# SPDX-License-Identifier: ISC + +# !! WARNING !! +# Do not modify this file directly! +# !! WARNING !! +# +# It is generated from: ../workflows-src/check-functionality.yml +# using the script from https://github.com/mithro/actions-includes@main on: push: pull_request: - +name: Functionality jobs: - - Functionality: - runs-on: ubuntu-latest - + functionality: + runs-on: ubuntu-20.04 strategy: matrix: - antlr_runtime_type: [static, shared] + antlr_runtime_type: + - static + - shared include: - - { python-version: 3.5, TOXENV: py35 } - - { python-version: 3.6, TOXENV: py36 } - - { python-version: 3.7, TOXENV: py37 } - - { python-version: 3.8, TOXENV: py38 } - - { python-version: 3.9, TOXENV: py39 } + - python-version: 3.5 + TOXENV: py35 + - python-version: 3.6 + TOXENV: py36 + - python-version: 3.7 + TOXENV: py37 + - python-version: 3.8 + TOXENV: py38 + - python-version: 3.9 + TOXENV: py39 fail-fast: false - name: Functionality on Python ${{ matrix.python-version }} (with ${{ matrix.antlr_runtime_type}} antlr) - steps: - - run: | - echo "${{ github.repository }}/.github/workflows/system-setup@${{ github.sha }}" - - - uses: mithro/fasm/.github/workflows/system-setup@setupcfg + - uses: mithro/actions-includes@main + if: runner.os == 'Linux' + continue-on-error: false + with: + workflow: .github/workflows/check-functionality.yml + - name: 🐍 Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 with: - development-tools: true python-version: ${{ matrix.python-version }} - - - name: Run Tests + - name: Install latest pip + run: | + pip install -U pip + - name: Install package's system dependencies (Ubuntu) run: | - ANTLR4_RUNTIME_TYPE=${{ matrix.antlr_runtime_type }} tox -e ${{ matrix.TOXENV }} + sudo apt-get update + sudo apt-get install -y cmake default-jre-headless uuid-dev libantlr4-runtime-dev + - name: 🧰 Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + submodules: true + - name: Install developer tooling's system dependencies (Ubuntu) + run: | + sudo apt-get update + sudo apt-get install -y clang-format + - name: Install development tools + run: | + pip install -r requirements.txt + - name: Run Tests + run: ANTLR4_RUNTIME_TYPE=${{ matrix.antlr_runtime_type }} tox -e ${{ matrix.TOXENV }} diff --git a/.github/workflows/check-install.yml b/.github/workflows/check-install.yml index 9fa52e42..fcaf9e62 100644 --- a/.github/workflows/check-install.yml +++ b/.github/workflows/check-install.yml @@ -1,115 +1,770 @@ -name: Install from +# Copyright (C) 2017-2021 The SymbiFlow Authors. +# +# Use of this source code is governed by a ISC-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/ISC +# +# SPDX-License-Identifier: ISC + +# !! WARNING !! +# Do not modify this file directly! +# !! WARNING !! +# +# It is generated from: ../workflows-src/check-install.yml +# using the script from https://github.com/mithro/actions-includes@main on: push: pull_request: - - +name: Install from jobs: - GitHub: strategy: matrix: - os: [windows-latest, macos-latest, ubuntu-latest] + os: + - windows-latest + - macos-latest + - ubuntu-20.04 fail-fast: false - runs-on: ${{ matrix.os }} - steps: - - uses: '${{ github.repository }}/.github/workflows/system-setup@${{ github.sha }}' + - uses: mithro/actions-includes@main + if: runner.os == 'Linux' + continue-on-error: false with: - os: ${{ matrix.os }} - git-checkout: false - - - name: Test installation + workflow: .github/workflows/check-install.yml + - name: 🐍 Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: 3.x + - name: Install latest pip + run: | + pip install -U pip + - name: Install package's system dependencies (Ubuntu) + if: ${{ startsWith(matrix.os, 'ubuntu') }} + run: | + sudo apt-get update + sudo apt-get install -y cmake default-jre-headless uuid-dev libantlr4-runtime-dev + - name: Install package's system dependencies (Mac OS X) + if: ${{ startsWith(matrix.os, 'macos') }} + run: | + true + - name: Install package's system dependencies (Windows) + if: ${{ startsWith(matrix.os, 'windows') }} + run: | + true + - name: Installing directly from GitHub shell: bash run: | - pip install git+https://github.com/${GITHUB_REPOSITORY}.git@${GITHUB_SHA}#egg=fasm - - - name: Run smoke test + pip install --verbose git+https://github.com/${GITHUB_REPOSITORY}.git@${GITHUB_SHA}#egg=fasm + - name: Smoke Test - Run fasm tool run: | fasm --help + - name: Smoke Test - Import fasm module + shell: python + run: | + import fasm + - name: Smoke Test - Print fasm version info + run: | + #!/usr/bin/env python + + import fasm.version + + l = [] + + print() + print(' FASM library version info') + print('='*75) -# - name: Run tests against installed version -# run: | -# cd tests; python test_simple.py + kl = max(len(k) for k in dir(fasm.version)) + for k in dir(fasm.version): + if '__' in k: + continue + v = getattr(fasm.version, k) + if isinstance(v, str) and '\n' in v: + l.append((k,v)) + else: + print(" {!s}: {!r}".format(k.rjust(kl), v)) + for k, v in l: + print() + print(k) + print('-'*75) + print(v) + print('-'*75) + shell: python + - name: Getting the tests + run: | + if [ -d tests ]; then + echo "::group::Using existing tests" + ls -l tests + echo "::endgroup::" + else + echo "::group::Event info" + cat ${GITHUB_EVENT_PATH} + echo "::endgroup::" + echo "::group::GitHub info" + echo "GITHUB_REPOSITORY: ${GITHUB_REPOSITORY}" + echo " GITHUB_ACTOR: ${GITHUB_ACTOR}" + echo " GITHUB_REF: ${GITHUB_REF}" + echo " GITHUB_BASE_REF: ${GITHUB_BASE_REF}" + echo " GITHUB_HEAD_REF: ${GITHUB_HEAD_REF}" + echo " GITHUB_SHA: ${GITHUB_SHA}" + echo "::endgroup::" + echo "::group::Downloading tests from ${GITHUB_REPOSITORY}" + set -x + mkdir .checkout-tests + cd .checkout-tests + git init + git config core.sparseCheckout true + if [ -f .git/info/sparse-checkout ]; then + rm .git/info/sparse-checkout + fi + echo "tests/*" >> .git/info/sparse-checkout + echo "examples/*" >> .git/info/sparse-checkout + git remote add origin ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git + git fetch --all + git remote -v + if [ ! -z "${GITHUB_REF}" ]; then + git fetch --refmap='' origin ${GITHUB_REF}:refs/remotes/origin/merge || true + fi + if [ ! -z "${GITHUB_BASE_REF}" ]; then + git fetch --refmap='' origin refs/heads/${GITHUB_BASE_REF}:refs/remotes/origin/base || true + fi + if [ ! -z "${GITHUB_HEAD_REF}" ]; then + git fetch --refmap='' origin refs/heads/${GITHUB_HEAD_REF}:refs/remotes/origin/head || true + fi + git remote show origin + git branch -v -a + + git show-ref ${GITHUB_SHA} || true + git rev-parse --verify "sha^${GITHUB_SHA}" || true + + git fetch -q https://github.com/SymbiFlow/fasm.git ${GITHUB_SHA} + git rev-parse FETCH_HEAD + git checkout ${GITHUB_SHA} + for i in *; do + cp -rvf $i .. + done + cd .. + echo "::endgroup::" + fi + shell: bash + - name: List Tests + shell: bash + run: | + echo "::group::Top directory" + ls -l tests + echo "::endgroup::" + echo "::group::Files found" + find tests -type f | sort + echo "::endgroup::" + - name: Run Tests + shell: bash + run: | + cd tests + python test_simple.py Checkout: strategy: matrix: - os: [windows-latest, macos-latest, ubuntu-latest] + os: + - windows-latest + - macos-latest + - ubuntu-20.04 cmd: - - python setup.py install - - pip install . - - pip install -e . # Editable install + - python setup.py install + - pip install --verbose . + - pip install --verbose -e . fail-fast: false - - name: Checkout with '${{ matrix.cmd }}' + name: Checkout with '${{ matrix.cmd }}' (${{ matrix.os }}) runs-on: ${{ matrix.os }} - steps: - - uses: '${{ github.repository }}/.github/workflows/system-setup@${{ github.sha }}' + - uses: mithro/actions-includes@main + if: runner.os == 'Linux' + continue-on-error: false with: - os: ${{ matrix.os }} - - - name: Install using '${{ matrix.cmd }}' + workflow: .github/workflows/check-install.yml + - name: 🐍 Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: 3.x + - name: Install latest pip + run: | + pip install -U pip + - name: Install package's system dependencies (Ubuntu) + if: ${{ startsWith(matrix.os, 'ubuntu') }} + run: | + sudo apt-get update + sudo apt-get install -y cmake default-jre-headless uuid-dev libantlr4-runtime-dev + - name: Install package's system dependencies (Mac OS X) + if: ${{ startsWith(matrix.os, 'macos') }} + run: | + true + - name: Install package's system dependencies (Windows) + if: ${{ startsWith(matrix.os, 'windows') }} + run: | + true + - name: 🧰 Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + submodules: true + - name: Installing using '${{ matrix.cmd }}' run: | ${{ matrix.cmd }} - - - name: Run smoke test + - name: Smoke Test - Run fasm tool run: | fasm --help + - name: Smoke Test - Import fasm module + shell: python + run: | + import fasm + - name: Smoke Test - Print fasm version info + run: | + #!/usr/bin/env python + + import fasm.version + + l = [] + + print() + print(' FASM library version info') + print('='*75) + + kl = max(len(k) for k in dir(fasm.version)) + for k in dir(fasm.version): + if '__' in k: + continue + v = getattr(fasm.version, k) + if isinstance(v, str) and '\n' in v: + l.append((k,v)) + else: + print(" {!s}: {!r}".format(k.rjust(kl), v)) - - name: Run tests against installed version + for k, v in l: + print() + print(k) + print('-'*75) + print(v) + print('-'*75) + shell: python + - name: Getting the tests run: | - cd tests; python test_simple.py + if [ -d tests ]; then + echo "::group::Using existing tests" + ls -l tests + echo "::endgroup::" + else + echo "::group::Event info" + cat ${GITHUB_EVENT_PATH} + echo "::endgroup::" + echo "::group::GitHub info" + echo "GITHUB_REPOSITORY: ${GITHUB_REPOSITORY}" + echo " GITHUB_ACTOR: ${GITHUB_ACTOR}" + echo " GITHUB_REF: ${GITHUB_REF}" + echo " GITHUB_BASE_REF: ${GITHUB_BASE_REF}" + echo " GITHUB_HEAD_REF: ${GITHUB_HEAD_REF}" + echo " GITHUB_SHA: ${GITHUB_SHA}" + echo "::endgroup::" + echo "::group::Downloading tests from ${GITHUB_REPOSITORY}" + set -x + mkdir .checkout-tests + cd .checkout-tests + git init + git config core.sparseCheckout true + if [ -f .git/info/sparse-checkout ]; then + rm .git/info/sparse-checkout + fi + echo "tests/*" >> .git/info/sparse-checkout + echo "examples/*" >> .git/info/sparse-checkout + git remote add origin ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git + git fetch --all + git remote -v + if [ ! -z "${GITHUB_REF}" ]; then + git fetch --refmap='' origin ${GITHUB_REF}:refs/remotes/origin/merge || true + fi + if [ ! -z "${GITHUB_BASE_REF}" ]; then + git fetch --refmap='' origin refs/heads/${GITHUB_BASE_REF}:refs/remotes/origin/base || true + fi + if [ ! -z "${GITHUB_HEAD_REF}" ]; then + git fetch --refmap='' origin refs/heads/${GITHUB_HEAD_REF}:refs/remotes/origin/head || true + fi + git remote show origin + git branch -v -a - Wheel: + git show-ref ${GITHUB_SHA} || true + git rev-parse --verify "sha^${GITHUB_SHA}" || true + + git fetch -q https://github.com/SymbiFlow/fasm.git ${GITHUB_SHA} + git rev-parse FETCH_HEAD + git checkout ${GITHUB_SHA} + for i in *; do + cp -rvf $i .. + done + cd .. + echo "::endgroup::" + fi + shell: bash + - name: List Tests + shell: bash + run: | + echo "::group::Top directory" + ls -l tests + echo "::endgroup::" + echo "::group::Files found" + find tests -type f | sort + echo "::endgroup::" + - name: Run Tests + shell: bash + run: | + cd tests + python test_simple.py + make-env: strategy: matrix: - os: [windows-latest, macos-latest, ubuntu-latest] + os: + - windows-latest + - macos-latest + - ubuntu-20.04 fail-fast: false - + name: make-env (Conda) runs-on: ${{ matrix.os }} - steps: - - uses: '${{ github.repository }}/.github/workflows/system-setup@${{ github.sha }}' + - uses: mithro/actions-includes@main + if: runner.os == 'Linux' + continue-on-error: false with: - os: ${{ matrix.os }} - + workflow: .github/workflows/check-install.yml + - name: 🐍 Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: 3.x + - name: Install latest pip + run: | + pip install -U pip + - name: Install package's system dependencies (Ubuntu) + if: ${{ startsWith(matrix.os, 'ubuntu') }} + run: | + sudo apt-get update + sudo apt-get install -y cmake default-jre-headless uuid-dev libantlr4-runtime-dev + - name: Install package's system dependencies (Mac OS X) + if: ${{ startsWith(matrix.os, 'macos') }} + run: | + true + - name: Install package's system dependencies (Windows) + if: ${{ startsWith(matrix.os, 'windows') }} + run: | + true + - name: 🧰 Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + submodules: true + - name: Run tests + run: | + make test + BuildSdist: + strategy: + matrix: + os: + - ubuntu-20.04 + fail-fast: false + runs-on: ${{ matrix.os }} + steps: + - uses: mithro/actions-includes@main + if: runner.os == 'Linux' + continue-on-error: false + with: + workflow: .github/workflows/check-install.yml + - name: 🐍 Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: 3.x + - name: Install latest pip + run: | + pip install -U pip + - name: Install package's system dependencies (Ubuntu) + if: ${{ startsWith(matrix.os, 'ubuntu') }} + run: | + sudo apt-get update + sudo apt-get install -y cmake default-jre-headless uuid-dev libantlr4-runtime-dev + - name: Install package's system dependencies (Mac OS X) + if: ${{ startsWith(matrix.os, 'macos') }} + run: | + true + - name: Install package's system dependencies (Windows) + if: ${{ startsWith(matrix.os, 'windows') }} + run: | + true + - name: Install packaging tooling + run: | + pip install twine auditwheel build + - name: 🧰 Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + submodules: true - name: Build wheel run: | - python setup.py bdist_wheel - + python -m build --sdist - name: Upload wheel uses: actions/upload-artifact@v2 with: - name: fasm + name: fasm-sdist path: dist - - - name: Install wheel + sdist: + strategy: + matrix: + os: + - windows-latest + - macos-latest + - ubuntu-20.04 + fail-fast: false + runs-on: ${{ matrix.os }} + needs: BuildSdist + steps: + - uses: mithro/actions-includes@main + if: runner.os == 'Linux' + continue-on-error: false + with: + workflow: .github/workflows/check-install.yml + - name: 🐍 Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: 3.x + - name: Install latest pip run: | - pip install dist/*.whl - - - name: Run smoke test + pip install -U pip + - name: Install package's system dependencies (Ubuntu) + if: ${{ startsWith(matrix.os, 'ubuntu') }} + run: | + sudo apt-get update + sudo apt-get install -y cmake default-jre-headless uuid-dev libantlr4-runtime-dev + - name: Install package's system dependencies (Mac OS X) + if: ${{ startsWith(matrix.os, 'macos') }} + run: | + true + - name: Install package's system dependencies (Windows) + if: ${{ startsWith(matrix.os, 'windows') }} + run: | + true + - name: Download sdist + uses: actions/download-artifact@v2 + with: + name: fasm-sdist + path: dist + - name: Installing using the sdist + shell: bash + run: | + ls -l dist/* + pip install --verbose dist/* + - name: Smoke Test - Run fasm tool run: | fasm --help + - name: Smoke Test - Import fasm module + shell: python + run: | + import fasm + - name: Smoke Test - Print fasm version info + run: | + #!/usr/bin/env python + + import fasm.version + + l = [] - - name: Run tests against installed version + print() + print(' FASM library version info') + print('='*75) + + kl = max(len(k) for k in dir(fasm.version)) + for k in dir(fasm.version): + if '__' in k: + continue + v = getattr(fasm.version, k) + if isinstance(v, str) and '\n' in v: + l.append((k,v)) + else: + print(" {!s}: {!r}".format(k.rjust(kl), v)) + + for k, v in l: + print() + print(k) + print('-'*75) + print(v) + print('-'*75) + shell: python + - name: Getting the tests run: | - cd tests; python test_simple.py + if [ -d tests ]; then + echo "::group::Using existing tests" + ls -l tests + echo "::endgroup::" + else + echo "::group::Event info" + cat ${GITHUB_EVENT_PATH} + echo "::endgroup::" + echo "::group::GitHub info" + echo "GITHUB_REPOSITORY: ${GITHUB_REPOSITORY}" + echo " GITHUB_ACTOR: ${GITHUB_ACTOR}" + echo " GITHUB_REF: ${GITHUB_REF}" + echo " GITHUB_BASE_REF: ${GITHUB_BASE_REF}" + echo " GITHUB_HEAD_REF: ${GITHUB_HEAD_REF}" + echo " GITHUB_SHA: ${GITHUB_SHA}" + echo "::endgroup::" + echo "::group::Downloading tests from ${GITHUB_REPOSITORY}" + set -x + mkdir .checkout-tests + cd .checkout-tests + git init + git config core.sparseCheckout true + if [ -f .git/info/sparse-checkout ]; then + rm .git/info/sparse-checkout + fi + echo "tests/*" >> .git/info/sparse-checkout + echo "examples/*" >> .git/info/sparse-checkout + git remote add origin ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git + git fetch --all + git remote -v + if [ ! -z "${GITHUB_REF}" ]; then + git fetch --refmap='' origin ${GITHUB_REF}:refs/remotes/origin/merge || true + fi + if [ ! -z "${GITHUB_BASE_REF}" ]; then + git fetch --refmap='' origin refs/heads/${GITHUB_BASE_REF}:refs/remotes/origin/base || true + fi + if [ ! -z "${GITHUB_HEAD_REF}" ]; then + git fetch --refmap='' origin refs/heads/${GITHUB_HEAD_REF}:refs/remotes/origin/head || true + fi + git remote show origin + git branch -v -a - make-env: + git show-ref ${GITHUB_SHA} || true + git rev-parse --verify "sha^${GITHUB_SHA}" || true + + git fetch -q https://github.com/SymbiFlow/fasm.git ${GITHUB_SHA} + git rev-parse FETCH_HEAD + git checkout ${GITHUB_SHA} + for i in *; do + cp -rvf $i .. + done + cd .. + echo "::endgroup::" + fi + shell: bash + - name: List Tests + shell: bash + run: | + echo "::group::Top directory" + ls -l tests + echo "::endgroup::" + echo "::group::Files found" + find tests -type f | sort + echo "::endgroup::" + - name: Run Tests + shell: bash + run: | + cd tests + python test_simple.py + BuildWheel: strategy: matrix: - os: [windows-latest, macos-latest, ubuntu-latest] - - name: make-env (Conda) + os: + - windows-latest + - macos-latest + - ubuntu-20.04 + fail-fast: false runs-on: ${{ matrix.os }} - steps: - - uses: '${{ github.repository }}/.github/workflows/checkout@${{ github.sha }}' + - uses: mithro/actions-includes@main + if: runner.os == 'Linux' + continue-on-error: false + with: + workflow: .github/workflows/check-install.yml + - name: 🐍 Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: 3.x + - name: Install latest pip + run: | + pip install -U pip + - name: Install package's system dependencies (Ubuntu) + if: ${{ startsWith(matrix.os, 'ubuntu') }} + run: | + sudo apt-get update + sudo apt-get install -y cmake default-jre-headless uuid-dev libantlr4-runtime-dev + - name: Install package's system dependencies (Mac OS X) + if: ${{ startsWith(matrix.os, 'macos') }} + run: | + true + - name: Install package's system dependencies (Windows) + if: ${{ startsWith(matrix.os, 'windows') }} + run: | + true + - name: Install packaging tooling + run: | + pip install twine auditwheel build + - name: 🧰 Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + submodules: true + - name: Build wheel + run: | + python -m build --wheel + - name: Upload wheel + uses: actions/upload-artifact@v2 + with: + name: fasm-wheel-${{ matrix.os }} + path: dist + Wheel: + strategy: + matrix: + os: + - windows-latest + - macos-latest + - ubuntu-20.04 + fail-fast: false + runs-on: ${{ matrix.os }} + needs: BuildWheel + steps: + - uses: mithro/actions-includes@main + if: runner.os == 'Linux' + continue-on-error: false + with: + workflow: .github/workflows/check-install.yml + - name: 🐍 Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: 3.x + - name: Install latest pip + run: | + pip install -U pip + - name: Install package's system dependencies (Ubuntu) + if: ${{ startsWith(matrix.os, 'ubuntu') }} + run: | + sudo apt-get update + sudo apt-get install -y cmake default-jre-headless uuid-dev libantlr4-runtime-dev + - name: Install package's system dependencies (Mac OS X) + if: ${{ startsWith(matrix.os, 'macos') }} + run: | + true + - name: Install package's system dependencies (Windows) + if: ${{ startsWith(matrix.os, 'windows') }} + run: | + true + - name: Download wheel + uses: actions/download-artifact@v2 + with: + name: fasm-wheel-${{ matrix.os }} + path: dist + - name: Installing using the wheel + run: | + ls -l dist/* + pip install --verbose dist/*.whl + - name: Smoke Test - Run fasm tool + run: | + fasm --help + - name: Smoke Test - Import fasm module + shell: python + run: | + import fasm + - name: Smoke Test - Print fasm version info + run: | + #!/usr/bin/env python - - name: Run tests + import fasm.version + + l = [] + + print() + print(' FASM library version info') + print('='*75) + + kl = max(len(k) for k in dir(fasm.version)) + for k in dir(fasm.version): + if '__' in k: + continue + v = getattr(fasm.version, k) + if isinstance(v, str) and '\n' in v: + l.append((k,v)) + else: + print(" {!s}: {!r}".format(k.rjust(kl), v)) + + for k, v in l: + print() + print(k) + print('-'*75) + print(v) + print('-'*75) + shell: python + - name: Getting the tests + run: | + if [ -d tests ]; then + echo "::group::Using existing tests" + ls -l tests + echo "::endgroup::" + else + echo "::group::Event info" + cat ${GITHUB_EVENT_PATH} + echo "::endgroup::" + echo "::group::GitHub info" + echo "GITHUB_REPOSITORY: ${GITHUB_REPOSITORY}" + echo " GITHUB_ACTOR: ${GITHUB_ACTOR}" + echo " GITHUB_REF: ${GITHUB_REF}" + echo " GITHUB_BASE_REF: ${GITHUB_BASE_REF}" + echo " GITHUB_HEAD_REF: ${GITHUB_HEAD_REF}" + echo " GITHUB_SHA: ${GITHUB_SHA}" + echo "::endgroup::" + echo "::group::Downloading tests from ${GITHUB_REPOSITORY}" + set -x + mkdir .checkout-tests + cd .checkout-tests + git init + git config core.sparseCheckout true + if [ -f .git/info/sparse-checkout ]; then + rm .git/info/sparse-checkout + fi + echo "tests/*" >> .git/info/sparse-checkout + echo "examples/*" >> .git/info/sparse-checkout + git remote add origin ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git + git fetch --all + git remote -v + if [ ! -z "${GITHUB_REF}" ]; then + git fetch --refmap='' origin ${GITHUB_REF}:refs/remotes/origin/merge || true + fi + if [ ! -z "${GITHUB_BASE_REF}" ]; then + git fetch --refmap='' origin refs/heads/${GITHUB_BASE_REF}:refs/remotes/origin/base || true + fi + if [ ! -z "${GITHUB_HEAD_REF}" ]; then + git fetch --refmap='' origin refs/heads/${GITHUB_HEAD_REF}:refs/remotes/origin/head || true + fi + git remote show origin + git branch -v -a + + git show-ref ${GITHUB_SHA} || true + git rev-parse --verify "sha^${GITHUB_SHA}" || true + + git fetch -q https://github.com/SymbiFlow/fasm.git ${GITHUB_SHA} + git rev-parse FETCH_HEAD + git checkout ${GITHUB_SHA} + for i in *; do + cp -rvf $i .. + done + cd .. + echo "::endgroup::" + fi + shell: bash + - name: List Tests + shell: bash + run: | + echo "::group::Top directory" + ls -l tests + echo "::endgroup::" + echo "::group::Files found" + find tests -type f | sort + echo "::endgroup::" + - name: Run Tests + shell: bash run: | - make tests + cd tests + python test_simple.py diff --git a/.github/workflows/check-style.yml b/.github/workflows/check-style.yml index 4b3cc8c7..486fe42a 100644 --- a/.github/workflows/check-style.yml +++ b/.github/workflows/check-style.yml @@ -1,31 +1,63 @@ -name: Style +# Copyright (C) 2021 The SymbiFlow Authors. +# +# Use of this source code is governed by a ISC-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/ISC +# +# SPDX-License-Identifier: ISC + +# !! WARNING !! +# Do not modify this file directly! +# !! WARNING !! +# +# It is generated from: ../workflows-src/check-style.yml +# using the script from https://github.com/mithro/actions-includes@main on: push: pull_request: - +name: Style jobs: - Style: - runs-on: ubuntu-latest - + runs-on: ubuntu-20.04 steps: - - uses: '${{ github.repository }}/.github/workflows/system-setup@${{ github.sha }}' + - uses: mithro/actions-includes@main + if: runner.os == 'Linux' + continue-on-error: false with: - development-tools: true - + workflow: .github/workflows/check-style.yml + - name: 🐍 Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: 3.x + - name: Install latest pip + run: | + pip install -U pip + - name: Install package's system dependencies (Ubuntu) + run: | + sudo apt-get update + sudo apt-get install -y cmake default-jre-headless uuid-dev libantlr4-runtime-dev + - name: 🧰 Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + submodules: true + - name: Install developer tooling's system dependencies (Ubuntu) + run: | + sudo apt-get update + sudo apt-get install -y clang-format + - name: Install development tools + run: | + pip install -r requirements.txt - name: Check license headers run: make check-license - - name: Python style check run: | make format lint test $(git status --porcelain | wc -l) -eq 0 || { git diff; false; } - - name: Python script checks run: make check-python-scripts - - name: C++ style check - run: | + run: |- make format-cpp test $(git status --porcelain | wc -l) -eq 0 || { git diff; false; } diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml index 2b0c8500..f6584f9b 100644 --- a/.github/workflows/publish-to-pypi.yml +++ b/.github/workflows/publish-to-pypi.yml @@ -1,74 +1,128 @@ -name: PyPI +# Copyright (C) 2017-2021 The SymbiFlow Authors. +# +# Use of this source code is governed by a ISC-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/ISC +# +# SPDX-License-Identifier: ISC + +# !! WARNING !! +# Do not modify this file directly! +# !! WARNING !! +# +# It is generated from: ../workflows-src/publish-to-pypi.yml +# using the script from https://github.com/mithro/actions-includes@main on: push: pull_request: workflow_dispatch: - +name: PyPI jobs: - Source: name: Source - + runs-on: ubuntu-20.04 steps: - - uses: '${{ github.repository }}/.github/workflows/system-setup@${{ github.sha }}' + - uses: mithro/actions-includes@main + if: runner.os == 'Linux' + continue-on-error: false with: - packaging-tools: true - + workflow: .github/workflows/publish-to-pypi.yml + - name: 🐍 Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: 3.x + - name: Install latest pip + run: | + pip install -U pip + - name: Install package's system dependencies (Ubuntu) + run: | + sudo apt-get update + sudo apt-get install -y cmake default-jre-headless uuid-dev libantlr4-runtime-dev + - name: Install packaging tooling + run: | + pip install twine auditwheel build + - name: 🧰 Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + submodules: true - name: Install package run: | pip install -e . - - name: 🚧 Build distribution 📦 run: | - python setup.py sdist - - - name: Check distribution 📦 + python -m build --sdist . + - name: ✔︎ Check 📦 run: | - twine check dist/* - + for WHEEL in dist/*.whl; do + echo + echo "::group::Checking $WHEEL" + echo + python -m zipfile --list $WHEEL + echo + auditwheel show $WHEEL + echo + twine check $WHEEL + echo + echo "::endgroup::" + done - name: 📤 Publish source to Test PyPI env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_TEST_PASSWORD }} - if: env.TWINE_PASSWORD != null + if: ${{ env.TWINE_PASSWORD != null }} run: | - twine upload --skip-existing --verbose --repository testpypi dist/*.tar.gz - + twine upload --skip-existing --verbose --repository testpypi dist/* - name: 📤 Publish source to PyPI - if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'SymbiFlow/fasm' + if: ${{ github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'SymbiFlow/fasm' && env.TWINE_PASSWORD != null }} env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} run: | twine upload dist/* - Linux: strategy: matrix: include: - - { python-version: '3.6', manylinux-python: 'cp36-cp36m' } - - { python-version: '3.7', manylinux-python: 'cp37-cp37m' } - - { python-version: '3.8', manylinux-python: 'cp38-cp38' } - - { python-version: '3.9', manylinux-python: 'cp39-cp39' } + - python-version: '3.6' + manylinux-python: cp36-cp36m + - python-version: '3.7' + manylinux-python: cp37-cp37m + - python-version: '3.8' + manylinux-python: cp38-cp38 + - python-version: '3.9' + manylinux-python: cp39-cp39 fail-fast: false - - name: '${{ matrix.name }} • manylinux' - + name: ${{ matrix.python-version }} • manylinux + runs-on: ubuntu-latest steps: - - uses: '${{ github.repository }}/.github/workflows/system-setup@${{ github.sha }}' + - uses: mithro/actions-includes@main + if: runner.os == 'Linux' + continue-on-error: false + with: + workflow: .github/workflows/publish-to-pypi.yml + - name: 🐍 Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - packaging-tools: true - + - name: Install latest pip + run: | + pip install -U pip + - name: Install packaging tooling + run: | + pip install twine auditwheel build + - name: 🧰 Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + submodules: true - name: 🚧 Build distribution 📦 uses: RalfG/python-wheels-manylinux-build@v0.3.3-manylinux2010_x86_64 with: - build-requirements: 'cython' - pre-build-command: 'bash .github/workflows/manylinux-install-cmake.sh' + build-requirements: cython + pre-build-command: bash .github/workflows/manylinux-install-cmake.sh python-versions: ${{ matrix.manylinux-python }} - #pip-wheel-args: '-w ./dist --no-deps --verbose' - - name: List distribution 📦 run: | # Fix permissions @@ -82,8 +136,7 @@ jobs: # Remove the non-manylinux versions rm -v dist/*linux_x86_64*.whl ls -l dist/* - - - name: Check distribution 📦 + - name: ✔︎ Check 📦 run: | for WHEEL in dist/*.whl; do echo @@ -97,67 +150,101 @@ jobs: echo echo "::endgroup::" done - - name: 📤 Publish wheels to Test PyPI env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_TEST_PASSWORD }} - if: env.TWINE_PASSWORD != null + if: ${{ env.TWINE_PASSWORD != null }} run: | - twine upload --skip-existing --verbose --repository testpypi dist/*.whl - - - name: 📤 Publish wheels to PyPI - if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'SymbiFlow/fasm' + twine upload --skip-existing --verbose --repository testpypi dist/* + - name: 📤 Publish source to PyPI + if: ${{ github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'SymbiFlow/fasm' && env.TWINE_PASSWORD != null }} env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} run: | - twine upload dist/*.whl - + twine upload dist/* MacAndWindows: strategy: matrix: - os: [windows-latest, macos-latest] - python-version: [ '3.6', '3.7', '3.8', '3.9', 'pypy-3.6', 'pypy-3.7' ] + os: + - windows-latest + - macos-latest + python-version: + - '3.6' + - '3.7' + - '3.8' + - '3.9' + - pypy-3.6 + - pypy-3.7 fail-fast: false - name: ${{ matrix.python-version }} • ${{ matrix.os }} runs-on: ${{ matrix.os }} - steps: - - uses: '${{ github.repository }}/.github/workflows/system-setup@${{ github.sha }}' + - uses: mithro/actions-includes@main + if: runner.os == 'Linux' + continue-on-error: false + with: + workflow: .github/workflows/publish-to-pypi.yml + - name: 🐍 Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 with: - os: ${{ matrix.os }} python-version: ${{ matrix.python-version }} - packaging-tools: true - + - name: Install latest pip + run: | + pip install -U pip + - name: Install package's system dependencies (Ubuntu) + if: ${{ startsWith(matrix.os, 'ubuntu') }} + run: | + sudo apt-get update + sudo apt-get install -y cmake default-jre-headless uuid-dev libantlr4-runtime-dev + - name: Install package's system dependencies (Mac OS X) + if: ${{ startsWith(matrix.os, 'macos') }} + run: | + true + - name: Install package's system dependencies (Windows) + if: ${{ startsWith(matrix.os, 'windows') }} + run: | + true + - name: Install packaging tooling + run: | + pip install twine auditwheel build + - name: 🧰 Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + submodules: true - name: Install package run: | pip install -e . - - name: 🚧 Build distribution 📦 run: | python setup.py bdist_wheel - - - name: Check distribution 📦 - shell: bash + - name: ✔︎ Check 📦 run: | - python -m zipfile -l dist/* - echo - twine check dist/* - + for WHEEL in dist/*.whl; do + echo + echo "::group::Checking $WHEEL" + echo + python -m zipfile --list $WHEEL + echo + auditwheel show $WHEEL + echo + twine check $WHEEL + echo + echo "::endgroup::" + done - name: 📤 Publish wheels to Test PyPI env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_TEST_PASSWORD }} - if: env.TWINE_PASSWORD != null + if: ${{ env.TWINE_PASSWORD != null }} run: | - twine upload --skip-existing --verbose --repository testpypi dist/*.whl - - - name: 📤 Publish wheels to PyPI - if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'SymbiFlow/fasm' + twine upload --skip-existing --verbose --repository testpypi dist/* + - name: 📤 Publish source to PyPI + if: ${{ github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'SymbiFlow/fasm' && env.TWINE_PASSWORD != null }} env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} run: | - twine upload dist/*.whl + twine upload dist/* diff --git a/Makefile b/Makefile index b9ed7321..0464a064 100644 --- a/Makefile +++ b/Makefile @@ -55,6 +55,13 @@ install: setup.py | $(CONDA_ENV_PYTHON) .PHONY: install +# Info about the environment +info: + @make --no-print-directory env-info + @echo + @$(IN_CONDA_ENV) python -W ignore .github/actions/download-and-run-tests/fasm-version.py + +.PHONY: info # Build/install locally rather than inside the environment. # ------------------------------------------------------------------------ @@ -120,6 +127,17 @@ format-cpp: .PHONY: format-cpp +# Format the GitHub workflow files +GHA_WORKFLOW_SRCS = $(wildcard .github/workflows-src/*.yml) +GHA_WORKFLOW_OUTS = $(addprefix .github/workflows/,$(notdir $(GHA_WORKFLOW_SRCS))) + +.github/workflows/%.yml: .github/workflows-src/%.yml $(CONDA_ENV_PYTHON) + $(IN_CONDA_ENV) python -m actions_includes $< $@ + +format-gha: $(GHA_WORKFLOW_OUTS) + echo $(GHA_WORKFLOW_OUTS) + @true + # Format all the files! format: format-py format-cpp true diff --git a/requirements.txt b/requirements.txt index b0780b84..a238cc05 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,3 +11,5 @@ pytest tox twine yapf==0.24.0 +# CI expansion +git+https://github.com/mithro/actions-includes.git#egg=actions-includes From 60dd8fb69987db8251de729f5dc26bdfbdad2ea7 Mon Sep 17 00:00:00 2001 From: Tim 'mithro' Ansell Date: Sun, 7 Mar 2021 18:11:08 -0800 Subject: [PATCH 12/12] DNM: Use the includes from mithro/actions. Signed-off-by: Tim 'mithro' Ansell --- .github/workflows-src/publish-to-pypi.yml | 92 +++------ .github/workflows/manylinux-install-cmake.sh | 2 - .github/workflows/publish-to-pypi.yml | 193 ++++++++++++------- 3 files changed, 142 insertions(+), 145 deletions(-) diff --git a/.github/workflows-src/publish-to-pypi.yml b/.github/workflows-src/publish-to-pypi.yml index 2a1e4eae..362ba14e 100644 --- a/.github/workflows-src/publish-to-pypi.yml +++ b/.github/workflows-src/publish-to-pypi.yml @@ -13,6 +13,7 @@ on: pull_request: workflow_dispatch: + jobs: Source: @@ -20,96 +21,47 @@ jobs: runs-on: ubuntu-20.04 steps: - - includes: /system-setup - with: - packaging-tools: true - - - name: Install package - run: | - pip install -e . - - - name: 🚧 Build distribution 📦 - run: | - python -m build --sdist . + - includes: /checkout - - includes: /upload-to-pypi + - includes: mithro/actions/includes/python/publish-to-pypi-src@main with: - root_repo: SymbiFlow/fasm - type: source + root_user: SymbiFlow - Linux: + Wheels-Linux: strategy: matrix: - include: - - { python-version: '3.6', manylinux-python: 'cp36-cp36m' } - - { python-version: '3.7', manylinux-python: 'cp37-cp37m' } - - { python-version: '3.8', manylinux-python: 'cp38-cp38' } - - { python-version: '3.9', manylinux-python: 'cp39-cp39' } + python-version: [ '3.6', '3.7', '3.8', '3.9' ] fail-fast: false name: '${{ matrix.python-version }} • manylinux' - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - - includes: /system-setup - with: - python-version: ${{ matrix.python-version }} - system-dependencies: false - packaging-tools: true - - - name: 🚧 Build distribution 📦 - uses: RalfG/python-wheels-manylinux-build@v0.3.3-manylinux2010_x86_64 - with: - build-requirements: 'cython' - pre-build-command: 'bash .github/workflows/manylinux-install-cmake.sh' - python-versions: ${{ matrix.manylinux-python }} - #pip-wheel-args: '-w ./dist --no-deps --verbose' - - - name: List distribution 📦 - run: | - # Fix permissions - echo "::group::Fixing permission" - ls -l dist/* - echo $USER - whoami - sudo chown -R $USER dist - ls -l dist/* - echo "::endgroup::" - # Remove the non-manylinux versions - rm -v dist/*linux_x86_64*.whl - ls -l dist/* + - includes: /checkout - - includes: /upload-to-pypi + - includes: mithro/actions/includes/python/publish-to-pypi-wheels-bin-linux@main with: - root_repo: SymbiFlow/fasm - type: wheels + pre-build-command: bash .github/workflows/manylinux-install-cmake.sh + build-requirements: cython + system-packages: java-1.8.0-openjdk uuid uuid-devel libuuid libuuid-devel git + python-version: ${{ matrix.python-version }} + package-path: '' + root_user: SymbiFlow - MacAndWindows: + Wheels-Other: strategy: matrix: os: [windows-latest, macos-latest] - python-version: [ '3.6', '3.7', '3.8', '3.9', 'pypy-3.6', 'pypy-3.7' ] + python-version: [ '3.6', '3.7', '3.8', '3.9' ] fail-fast: false - name: ${{ matrix.python-version }} • ${{ matrix.os }} + name: '${{ matrix.python-version }} • ${{ matrix.os }}' runs-on: ${{ matrix.os }} steps: - - includes: /system-setup - with: - os: ${{ matrix.os }} - python-version: ${{ matrix.python-version }} - packaging-tools: true - - - name: Install package - run: | - pip install -e . + - includes: /checkout - - name: 🚧 Build distribution 📦 - run: | - python setup.py bdist_wheel - - - includes: /upload-to-pypi + - includes: mithro/actions/includes/python/publish-to-pypi-wheels-bin-other@main with: - root_repo: SymbiFlow/fasm - type: wheels + python-version: ${{ matrix.python-version }} + root_user: SymbiFlow diff --git a/.github/workflows/manylinux-install-cmake.sh b/.github/workflows/manylinux-install-cmake.sh index d78dab9b..1eb98be8 100755 --- a/.github/workflows/manylinux-install-cmake.sh +++ b/.github/workflows/manylinux-install-cmake.sh @@ -9,8 +9,6 @@ yum remove cmake -y # Add in curl yum install wget -y -yum install java-1.8.0-openjdk uuid uuid-devel libuuid libuuid-devel -y - # Download new cmake wget https://github.com/Kitware/CMake/releases/download/v3.19.4/cmake-3.19.4-Linux-x86_64.sh -O /tmp/cmake.sh chmod a+x /tmp/cmake.sh diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml index f6584f9b..1e305602 100644 --- a/.github/workflows/publish-to-pypi.yml +++ b/.github/workflows/publish-to-pypi.yml @@ -28,6 +28,11 @@ jobs: continue-on-error: false with: workflow: .github/workflows/publish-to-pypi.yml + - name: 🧰 Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + submodules: true - name: 🐍 Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: @@ -35,38 +40,45 @@ jobs: - name: Install latest pip run: | pip install -U pip - - name: Install package's system dependencies (Ubuntu) - run: | - sudo apt-get update - sudo apt-get install -y cmake default-jre-headless uuid-dev libantlr4-runtime-dev - name: Install packaging tooling run: | - pip install twine auditwheel build - - name: 🧰 Checkout - uses: actions/checkout@v2 - with: - fetch-depth: 0 - submodules: true - - name: Install package + pip install twine wheel auditwheel build + - name: Install development tools run: | - pip install -e . + pip install -r requirements.txt - name: 🚧 Build distribution 📦 run: | - python -m build --sdist . + python -m build --sdist --no-isolation . - name: ✔︎ Check 📦 + shell: bash run: | - for WHEEL in dist/*.whl; do + shopt -s nullglob + for SRC in dist/*.zip; do echo - echo "::group::Checking $WHEEL" + echo "::group::Checking $SRC" echo - python -m zipfile --list $WHEEL + python -m zipfile --test $SRC + python -m zipfile --list $SRC echo - auditwheel show $WHEEL + twine check $SRC echo - twine check $WHEEL + echo "::endgroup::" + done + for SRC in dist/*.t*gz; do + echo + echo "::group::Checking $SRC" + echo + python -m tarfile --list $SRC + echo + twine check $SRC echo echo "::endgroup::" done + - name: Upload wheel + uses: actions/upload-artifact@v2 + with: + name: source-${{ runner.os }} + path: dist - name: 📤 Publish source to Test PyPI env: TWINE_USERNAME: __token__ @@ -75,54 +87,79 @@ jobs: run: | twine upload --skip-existing --verbose --repository testpypi dist/* - name: 📤 Publish source to PyPI - if: ${{ github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'SymbiFlow/fasm' && env.TWINE_PASSWORD != null }} + if: ${{ github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && startsWith('SymbiFlow', github.repository) && env.TWINE_PASSWORD != null }} env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} run: | twine upload dist/* - Linux: + Wheels-Linux: strategy: matrix: - include: - - python-version: '3.6' - manylinux-python: cp36-cp36m - - python-version: '3.7' - manylinux-python: cp37-cp37m - - python-version: '3.8' - manylinux-python: cp38-cp38 - - python-version: '3.9' - manylinux-python: cp39-cp39 + python-version: + - '3.6' + - '3.7' + - '3.8' + - '3.9' fail-fast: false name: ${{ matrix.python-version }} • manylinux - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - uses: mithro/actions-includes@main if: runner.os == 'Linux' continue-on-error: false with: workflow: .github/workflows/publish-to-pypi.yml - - name: 🐍 Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install latest pip - run: | - pip install -U pip - - name: Install packaging tooling - run: | - pip install twine auditwheel build - name: 🧰 Checkout uses: actions/checkout@v2 with: fetch-depth: 0 submodules: true + - name: Config Check + if: ${{ !startsWith(runner.os, 'Linux') }} + run: | + echo "Use" + echo + echo " SymbiFlow/actions/includes/python/publish-to-pypi-sheels-bin-other" + echo + echo "for Windows / Mac" + echo + exit 1 + - name: Install packaging tooling on runner + run: | + echo "$HOME/.local/bin" >> $GITHUB_PATH + export PATH=$HOME/.local/bin:$PATH + pip install -U pip twine wheel auditwheel + - name: Check packaging tooling on runner + run: | + echo $PATH + which pip + which twine + which wheel + which auditwheel + - name: Workout manylinux-version + id: manylinux + shell: python + env: + PYTHON_VERSION: ${{ matrix.python-version }} + run: | + import os + MANYLINUX = { + "3.6": "cp36-cp36m", + "3.7": "cp37-cp37m", + "3.8": "cp38-cp38", + "3.9": "cp39-cp39", + }[os.environ['PYTHON_VERSION']] + print("::set-output name=version::"+MANYLINUX) - name: 🚧 Build distribution 📦 uses: RalfG/python-wheels-manylinux-build@v0.3.3-manylinux2010_x86_64 with: build-requirements: cython + system-packages: java-1.8.0-openjdk uuid uuid-devel libuuid libuuid-devel git + python-versions: ${{ steps.manylinux.outputs.version }} pre-build-command: bash .github/workflows/manylinux-install-cmake.sh - python-versions: ${{ matrix.manylinux-python }} + package-path: '' + pip-wheel-args: -w ./dist --no-deps --verbose - name: List distribution 📦 run: | # Fix permissions @@ -137,19 +174,28 @@ jobs: rm -v dist/*linux_x86_64*.whl ls -l dist/* - name: ✔︎ Check 📦 + shell: bash run: | for WHEEL in dist/*.whl; do echo echo "::group::Checking $WHEEL" echo - python -m zipfile --list $WHEEL + python -m zipfile -t $WHEEL + python -m zipfile -l $WHEEL echo - auditwheel show $WHEEL + if [ "$(uname)" = "Linux" ]; then + auditwheel show $WHEEL + fi echo twine check $WHEEL echo echo "::endgroup::" done + - name: Upload wheel + uses: actions/upload-artifact@v2 + with: + name: wheels-${{ runner.os }} + path: dist - name: 📤 Publish wheels to Test PyPI env: TWINE_USERNAME: __token__ @@ -158,13 +204,13 @@ jobs: run: | twine upload --skip-existing --verbose --repository testpypi dist/* - name: 📤 Publish source to PyPI - if: ${{ github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'SymbiFlow/fasm' && env.TWINE_PASSWORD != null }} + if: ${{ github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && startsWith('SymbiFlow', github.repository) && env.TWINE_PASSWORD != null }} env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} run: | twine upload dist/* - MacAndWindows: + Wheels-Other: strategy: matrix: os: @@ -175,8 +221,6 @@ jobs: - '3.7' - '3.8' - '3.9' - - pypy-3.6 - - pypy-3.7 fail-fast: false name: ${{ matrix.python-version }} • ${{ matrix.os }} runs-on: ${{ matrix.os }} @@ -186,6 +230,21 @@ jobs: continue-on-error: false with: workflow: .github/workflows/publish-to-pypi.yml + - name: 🧰 Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + submodules: true + - name: Config Check + if: ${{ startsWith(runner.os, 'Linux') }} + run: | + echo "Use" + echo + echo " SymbiFlow/actions/includes/python/publish-to-pypi-sheels-bin-linux" + echo + echo "for Linux" + echo + exit 1 - name: 🐍 Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: @@ -193,47 +252,35 @@ jobs: - name: Install latest pip run: | pip install -U pip - - name: Install package's system dependencies (Ubuntu) - if: ${{ startsWith(matrix.os, 'ubuntu') }} - run: | - sudo apt-get update - sudo apt-get install -y cmake default-jre-headless uuid-dev libantlr4-runtime-dev - - name: Install package's system dependencies (Mac OS X) - if: ${{ startsWith(matrix.os, 'macos') }} - run: | - true - - name: Install package's system dependencies (Windows) - if: ${{ startsWith(matrix.os, 'windows') }} - run: | - true - name: Install packaging tooling run: | - pip install twine auditwheel build - - name: 🧰 Checkout - uses: actions/checkout@v2 - with: - fetch-depth: 0 - submodules: true - - name: Install package - run: | - pip install -e . + pip install twine wheel auditwheel build - name: 🚧 Build distribution 📦 run: | - python setup.py bdist_wheel + python -m build --wheel . - name: ✔︎ Check 📦 + shell: bash run: | for WHEEL in dist/*.whl; do echo echo "::group::Checking $WHEEL" echo - python -m zipfile --list $WHEEL + python -m zipfile -t $WHEEL + python -m zipfile -l $WHEEL echo - auditwheel show $WHEEL + if [ "$(uname)" = "Linux" ]; then + auditwheel show $WHEEL + fi echo twine check $WHEEL echo echo "::endgroup::" done + - name: Upload wheel + uses: actions/upload-artifact@v2 + with: + name: wheels-${{ runner.os }} + path: dist - name: 📤 Publish wheels to Test PyPI env: TWINE_USERNAME: __token__ @@ -242,7 +289,7 @@ jobs: run: | twine upload --skip-existing --verbose --repository testpypi dist/* - name: 📤 Publish source to PyPI - if: ${{ github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'SymbiFlow/fasm' && env.TWINE_PASSWORD != null }} + if: ${{ github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && startsWith('SymbiFlow', github.repository) && env.TWINE_PASSWORD != null }} env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}