Switch CI to GitHub Actions (#285)

* remove CircleCI!

* timeout

* HIVEMIND_THREADS!!!

* Add caching

* fix caching

* BUMP!!!

* Update main-ci.yml

* Check prebuilt p2pd

* Fix a few imports

* Compile protobuf explicitly in BuildPy (#290)

Co-authored-by: Max Ryabinin <mryabinin0@gmail.com>
Co-authored-by: justheuristic <justheuristic@gmail.com>

.circleci/config.yml

@@ -1,100 +0,0 @@
-version: 2.1
-
-parameters:
-  go-version:
-    type: string
-    default: 1.16.2
-
-jobs:
-  build-and-test-py37:
-    docker:
-      - image: circleci/python:3.7.10
-    steps:
-      - checkout
-      - restore_cache:
-          keys:
-            - py37-v1-{{ checksum "requirements.txt" }}-{{ checksum "requirements-dev.txt" }}
-            - v1-{{ checksum "requirements.txt" }}-{{ checksum "requirements-dev.txt" }}
-      - run: |
-          wget https://golang.org/dl/go<< pipeline.parameters.go-version >>.linux-amd64.tar.gz -O go.tar.gz
-          tar -C ~/ -xzf go.tar.gz
-          echo "export PATH=~/go/bin:$PATH" >> $BASH_ENV
-      - run: pip install -r requirements.txt
-      - run: pip install -r requirements-dev.txt
-      - save_cache:
-          key: py37-v1-{{ checksum "requirements.txt" }}-{{ checksum "requirements-dev.txt" }}
-          paths:
-            - '~/.cache/pip'
-      - run:
-          command: pip install -e . --install-option="--buildgo"
-          name: setup
-      - run:
-          command: pytest --cov=hivemind ./tests
-          name: tests
-      - run:
-          command: codecov
-          name: codecov
-
-  build-and-test-py38:
-    docker:
-      - image: circleci/python:3.8.1
-    steps:
-      - checkout
-      - restore_cache:
-          keys:
-            - py38-v1-{{ checksum "requirements.txt" }}-{{ checksum "requirements-dev.txt" }}
-      - run: |
-          wget https://golang.org/dl/go<< pipeline.parameters.go-version >>.linux-amd64.tar.gz -O go.tar.gz
-          tar -C ~/ -xzf go.tar.gz
-          echo "export PATH=~/go/bin:$PATH" >> $BASH_ENV
-      - run: pip install -r requirements.txt
-      - run: pip install -r requirements-dev.txt
-      - save_cache:
-          key: py38-v1-{{ checksum "requirements.txt" }}-{{ checksum "requirements-dev.txt" }}
-          paths:
-            - '~/.cache/pip'
-      - run:
-          command: pip install -e . --install-option="--buildgo"
-          name: setup
-      - run:
-          command: pytest --cov=hivemind ./tests
-          name: tests
-      - run:
-          command: codecov
-          name: codecov
-
-  build-and-test-py39:
-    docker:
-      - image: circleci/python:3.9.1
-    steps:
-      - checkout
-      - run: ulimit -n 4096 # temporary workaround for py39
-      - restore_cache:
-          keys:
-            - py39-v1-{{ checksum "requirements.txt" }}-{{ checksum "requirements-dev.txt" }}
-      - run: |
-          wget https://golang.org/dl/go<< pipeline.parameters.go-version >>.linux-amd64.tar.gz -O go.tar.gz
-          tar -C ~/ -xzf go.tar.gz
-          echo "export PATH=~/go/bin:$PATH" >> $BASH_ENV
-      - run: pip install -r requirements.txt
-      - run: pip install -r requirements-dev.txt
-      - save_cache:
-          key: py39-v1-{{ checksum "requirements.txt" }}-{{ checksum "requirements-dev.txt" }}
-          paths:
-            - '~/.cache/pip'
-      - run:
-          command: pip install -e . --install-option="--buildgo"
-          name: setup
-      - run:
-          command: pytest --cov=hivemind ./tests
-          name: tests
-      - run:
-          command: codecov
-          name: codecov
-
-workflows:
-  main:
-    jobs:
-      - build-and-test-py37
-      - build-and-test-py38
-      - build-and-test-py39

.github/workflows/run-tests.yml

@@ -0,0 +1,66 @@
+name: Tests
+
+on: [ push ]
+
+
+jobs:
+  run_tests:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [ 3.7, 3.8, 3.9 ]
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Cache dependencies
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: Key-v1-${{ matrix.python-version }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('requirements-dev.txt') }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+          pip install -r requirements-dev.txt
+      - name: Build hivemind
+        run: |
+          pip install .
+      - name: Test
+        run: |
+          cd tests
+          pytest --cov=hivemind 
+      - name: Upload to codecov
+        run: |
+          codecov
+
+  build_and_test_p2pd:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: Cache dependencies
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: Key-v1-${{ matrix.python-version }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('requirements-dev.txt') }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+          pip install -r requirements-dev.txt
+      - name: Build hivemind
+        run: |
+          pip install . --global-option=build_py --global-option="--buildgo"
+      - name: Test
+        run: |
+          cd tests
+          pytest -k "p2p" 

hivemind/__init__.py

@@ -5,4 +5,4 @@ from hivemind.server import *
 from hivemind.utils import *
 from hivemind.optim import *
 
-__version__ = "0.9.9.post1"
+__version__ = "0.9.10"

hivemind/utils/threading.py

@@ -12,7 +12,7 @@ def run_in_background(func: callable, *args, **kwargs) -> Future:
     """ run func(*args, **kwargs) in background and return Future for its outputs """
     global EXECUTOR_PID, GLOBAL_EXECUTOR
     if os.getpid() != EXECUTOR_PID:
-        GLOBAL_EXECUTOR = ThreadPoolExecutor(max_workers=float(os.environ.get("HIVEMIND_THREADS", 'inf')))
+        GLOBAL_EXECUTOR = ThreadPoolExecutor(max_workers=int(os.environ.get("HIVEMIND_THREADS", 128)))
         EXECUTOR_PID = os.getpid()
     return GLOBAL_EXECUTOR.submit(func, *args, **kwargs)
 

setup.py

@@ -11,8 +11,8 @@ import urllib.request
 
 from pkg_resources import parse_requirements, parse_version
 from setuptools import find_packages, setup
+from setuptools.command.build_py import build_py
 from setuptools.command.develop import develop
-from setuptools.command.install import install
 
 P2PD_VERSION = 'v0.3.1'
 P2PD_CHECKSUM = '15292b880c6b31f5b3c36084b3acc17f'
@@ -48,13 +48,14 @@ def proto_compile(output_path):
             file.truncate()
 
 
-def libp2p_build_install():
+def build_p2p_daemon():
     result = subprocess.run("go version", capture_output=True, shell=True).stdout.decode('ascii', 'replace')
     m = re.search(r'^go version go([\d.]+)', result)
 
     if m is None:
         raise FileNotFoundError('Could not find golang installation')
-    if parse_version(m.group(1)) < parse_version("1.13"):
+    version = parse_version(m.group(1))
+    if version < parse_version("1.13"):
         raise EnvironmentError(f'Newer version of go required: must be >= 1.13, found {version}')
 
     with tempfile.TemporaryDirectory() as tempdir:
@@ -72,7 +73,7 @@ def libp2p_build_install():
                                f' exited with status code: {result.returncode}')
 
 
-def libp2p_download_install():
+def download_p2p_daemon():
     install_path = os.path.join(here, 'hivemind', 'hivemind_cli')
     binary_path = os.path.join(install_path, 'p2pd')
     if not os.path.exists(binary_path) or md5(binary_path) != P2PD_CHECKSUM:
@@ -84,8 +85,8 @@ def libp2p_download_install():
             raise RuntimeError(f'Downloaded p2pd binary from {url} does not match with md5 checksum')
 
 
-class Install(install):
-    user_options = install.user_options + [('buildgo', None, "Builds p2pd from source")]
+class BuildPy(build_py):
+    user_options = build_py.user_options + [('buildgo', None, "Builds p2pd from source")]
 
     def initialize_options(self):
         super().initialize_options()
@@ -93,26 +94,19 @@ class Install(install):
 
     def run(self):
         if self.buildgo:
-            libp2p_build_install()
+            build_p2p_daemon()
         else:
-            libp2p_download_install()
-        proto_compile(os.path.join(self.build_lib, 'hivemind', 'proto'))
+            download_p2p_daemon()
+
         super().run()
 
+        proto_compile(os.path.join(self.build_lib, 'hivemind', 'proto'))
 
-class Develop(develop):
-    user_options = develop.user_options + [('buildgo', None, None)]
-
-    def initialize_options(self):
-        super().initialize_options()
-        self.buildgo = False
 
+class Develop(develop):
     def run(self):
-        if self.buildgo:
-            libp2p_build_install()
-        else:
-            libp2p_download_install()
-        proto_compile(os.path.join('hivemind', 'proto'))
+        self.reinitialize_command('build_py', build_lib=here)
+        self.run_command('build_py')
         super().run()
 
 
@@ -137,7 +131,7 @@ extras['all'] = extras['dev'] + extras['docs']
 setup(
     name='hivemind',
     version=version_string,
-    cmdclass={'install': Install, 'develop': Develop},
+    cmdclass={'build_py': BuildPy, 'develop': Develop},
     description='Decentralized deep learning in PyTorch',
     long_description='Decentralized deep learning in PyTorch. Built to train giant models on '
                      'thousands of volunteers across the world.',

tests/test_auth.py

@@ -1,12 +1,12 @@
 from datetime import datetime, timedelta
-from typing import Optional, Tuple
+from typing import Optional
 
 import pytest
 
 from hivemind.proto import dht_pb2
 from hivemind.proto.auth_pb2 import AccessToken
 from hivemind.utils.auth import AuthRPCWrapper, AuthRole, TokenAuthorizerBase
-from hivemind.utils.crypto import RSAPrivateKey, RSAPublicKey
+from hivemind.utils.crypto import RSAPrivateKey
 from hivemind.utils.logging import get_logger
 
 

tests/test_dht_schema.py

@@ -1,13 +1,11 @@
-import re
-
 import pytest
-from pydantic import BaseModel, StrictFloat, StrictInt, conint
+from pydantic import BaseModel, StrictInt, conint
 from typing import Dict
 
 import hivemind
 from hivemind.dht import get_dht_time
 from hivemind.dht.node import DHTNode, LOCALHOST
-from hivemind.dht.schema import BytesWithPublicKey, SchemaValidator, conbytes
+from hivemind.dht.schema import BytesWithPublicKey, SchemaValidator
 from hivemind.dht.validation import DHTRecord, RecordValidatorBase
 
 

tests/test_dht_validation.py

@@ -1,5 +1,4 @@
 import dataclasses
-from functools import partial
 from typing import Dict
 
 import pytest
@@ -10,7 +9,7 @@ from hivemind.dht.crypto import RSASignatureValidator
 from hivemind.dht.protocol import DHTProtocol
 from hivemind.dht.routing import DHTID
 from hivemind.dht.schema import BytesWithPublicKey, SchemaValidator
-from hivemind.dht.validation import DHTRecord, CompositeValidator, RecordValidatorBase
+from hivemind.dht.validation import DHTRecord, CompositeValidator
 
 
 class SchemaA(BaseModel):

tests/test_utils/__init__.py

@@ -6,6 +6,7 @@ import time
 import uuid
 from contextlib import asynccontextmanager
 from typing import NamedTuple
+from pkg_resources import resource_filename
 
 from multiaddr import Multiaddr, protocols
 
@@ -14,6 +15,7 @@ from hivemind.p2p.p2p_daemon_bindings.p2pclient import Client
 
 
 TIMEOUT_DURATION = 30  # seconds
+P2PD_PATH = resource_filename("hivemind", "hivemind_cli/p2pd")
 
 
 async def try_until_success(coro_func, timeout=TIMEOUT_DURATION):
@@ -57,7 +59,7 @@ class Daemon:
         self.f_log = open(self.log_filename, "wb")
 
     def _run(self):
-        cmd_list = ["hivemind/hivemind_cli/p2pd", f"-listen={str(self.control_maddr)}"]
+        cmd_list = [P2PD_PATH, f"-listen={str(self.control_maddr)}"]
         cmd_list += [f"-hostAddrs=/ip4/127.0.0.1/tcp/{find_open_port()}"]
         if self.enable_connmgr:
             cmd_list += ["-connManager=true", "-connLo=1", "-connHi=2", "-connGrace=0"]