Accelerate yacht train

.github/workflows/runTest.yml

@@ -8,24 +8,35 @@ jobs:
         shell: bash -el {0}
     steps:
     - uses: actions/checkout@v4
+
     - name: Python Linter
       uses: chartboost/ruff-action@v1
       with:
-        src: "yacht"
+        src: "src"
         # args: --select ALL 
     - uses: conda-incubator/setup-miniconda@v3
       with:
         miniconda-version: "latest"
         activate-environment: yacht_env
         environment-file: env/yacht_env.yml
+
     - name: install YACHT locally
-      run: pip install -e .
+      run: pip install .
+
+    - name: List contents of 'yacht' in site-packages
+      run: |
+        SITE_PACKAGES=$(python -c "import site; print(site.getsitepackages()[0])")
+        ls -R $SITE_PACKAGES/yacht
+
     - name: make training data
       run: yacht train --ref_file './tests/testdata/20_genomes_sketches.zip' --ksize 31 --prefix 'gtdb_ani_thresh_0.95' --ani_thresh 0.95 --outdir ./ --force
+
     - name: run YACHT
       run: yacht run --json ./gtdb_ani_thresh_0.95_config.json --sample_file './tests/testdata/sample.sig.zip' --significance 0.99 --min_coverage_list 1 0.6 0.2 0.1
+
     - name: unit-tests
       run: pytest tests/ --cov-report term-missing --cov-report xml:tests.xml --cov=yacht
+
     - name: Upload coverage reports to Codecov
       uses: codecov/codecov-action@v4
       with:

Makefile

@@ -0,0 +1,38 @@
+# Compiler and flags
+CXX = g++
+CXXFLAGS = -std=c++17 -Wall -w -O3 -Wsign-compare
+
+# Directories
+SRC_DIR = src/cpp
+BIN_DIR = src/yacht
+
+# Source files
+SRC_FILES = $(SRC_DIR)/main.cpp
+
+# Object files
+OBJ_FILES = $(SRC_FILES:.cpp=.o)
+
+# Target executable
+TARGET = $(BIN_DIR)/run_yacht_train_core
+
+# Default target
+all: $(TARGET)
+
+# Create the bin directory if it doesn't exist
+$(BIN_DIR):
+	echo "Creating directory: $(BIN_DIR)"
+	mkdir -p $(BIN_DIR)
+
+# build the object files
+$(OBJ_FILES): %.o: %.cpp
+	echo "Compiling: $<"
+	$(CXX) $(CXXFLAGS) -c $< -o $@ -lz
+
+# build the target executable
+$(TARGET): $(OBJ_FILES) | $(BIN_DIR)
+	echo "Linking to create executable: $(TARGET)"
+	$(CXX) $(CXXFLAGS) $(OBJ_FILES) -o $(TARGET) -lz -lpthread
+
+# clean up
+clean:
+	rm -f $(OBJ_FILES) $(TARGET)

README.md

@@ -302,7 +302,6 @@ The most important parameter of this script is `--ani_thresh`: this is average n
 | ------------------------------------- | ------------------------------------------------------------ |
 | _config.json                          | A JSON file stores the required information needed to run the next YACHT algorithm |
 | _manifest.tsv                         | A TSV file contains organisms and their relevant info after removing the similar ones |
-| _removed_orgs_to_corr_orgas_mapping.tsv   | A TSV file with two columns: removed organism names ('removed_org') and their similar genomes ('corr_orgs')| 
 
 #### Some pre-trained reference databases available on Zenodo  
 

build.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# Determine the platform
+OS_NAME="$(uname -s)"
+
+echo "Running build on Unix-based system: $OS_NAME"
+if [[ "$OS_NAME" == "Linux" || "$OS_NAME" == "Darwin" ]]; then
+  # Unix-based systems (Linux or macOS)
+  bash build_unix.sh
+elif [[ "$OS_NAME" == "MINGW"* || "$OS_NAME" == "CYGWIN"* || "$OS_NAME" == "MSYS"* ]]; then
+  # Windows-like environment detected, use batch file
+  cmd.exe /c build_windows.bat
+else
+  echo "Unsupported platform: $OS_NAME"
+  exit 1
+fi

build_unix.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+# Run the Makefile to compile the C++ code
+make

build_windows.bat

@@ -0,0 +1,21 @@
+@echo off
+
+REM Set up paths for directories
+set SRC_DIR=src\cpp
+set BIN_DIR=src\yacht
+
+REM Create bin directory if it doesn't exist
+if not exist %BIN_DIR% (
+    mkdir %BIN_DIR%
+)
+
+REM Compile the main.cpp file using g++ from MinGW or another suitable compiler
+g++ -std=c++17 -Wsign-compare -Wall -O3 -o %BIN_DIR%\run_yacht_train_core.exe %SRC_DIR%\main.cpp
+
+REM Check if compilation succeeded
+if %errorlevel% neq 0 (
+    echo Compilation failed!
+    exit /b %errorlevel%
+)
+
+echo Compilation successful. Executable created at %BIN_DIR%\run_yacht_train_core.exe

conda_recipe/meta.yaml

@@ -0,0 +1,78 @@
+{% set version = "1.3.0" %}
+
+package:
+  name: yacht
+  version: {{ version }}
+
+source:
+  url: https://github.com/KoslickiLab/YACHT/releases/download/v{{ version  }}/yacht-{{ version  }}.tar.gz
+  sha256: f5ea89cb4f4347bfd806b211f87ed402823ea1f603e755c05532c79ec75bb422
+
+build:
+  number: 0
+  script: "{{ PYTHON }} -m pip install . --no-deps --no-build-isolation --no-cache-dir -vvv"
+
+requirements:
+  build:
+    - {{ compiler('cxx') }}       # Adds platform-specific C++ compiler (g++, clang, MSVC)
+    - make                        # Ensures that Make is available (for Unix)
+    - python >3.6,<3.12           # Python version
+    - pip
+    - setuptools 
+
+  host:
+    - python >3.6,<3.12
+    - pip
+    - pandas
+    - pytaxonkit
+    - scipy
+    - sourmash
+    - loguru
+    - tqdm
+    - biom-format
+    - numpy >=1.22.4
+    - setuptools
+    - requests
+
+  run:
+    - python >3.6,<3.12
+    - sourmash >=4.8.3,<5
+    - scipy
+    - requests
+    - numpy >=1.22.4
+    - pandas
+    - scikit-learn
+    - codecov
+    - pytest
+    - pytest-cov
+    - loguru
+    - maturin >=1,<2
+    - tqdm
+    - biom-format
+    - pytaxonkit
+    - openpyxl
+    - ruff
+    - sourmash_plugin_branchwater
+
+
+test:
+  commands:
+    - yacht --help
+
+about:
+  home: https://github.com/KoslickiLab/YACHT
+  summary: 'YACHT is a mathematically rigorous hypothesis test for the presence or absence of organisms in a metagenomic sample, based on average nucleotide identity (ANI).'
+  license: MIT
+  license_family: MIT
+  license_file: LICENSE.txt
+  dev_url: https://github.com/KoslickiLab/YACHT
+  doc_url: https://github.com/KoslickiLab/YACHT/wiki
+
+extra:
+  skip-lints:
+    - should_use_compilers
+  identifiers:
+    - doi:10.1093/bioinformatics/btae047
+  recipe-maintainers:
+    - chunyuma
+    - dkoslicki

env/yacht_env.yml

@@ -4,11 +4,10 @@ channels:
   - bioconda
   - defaults
 dependencies:
-  - python>3.6
+  - python>3.6,<3.12
   - sourmash>=4.8.3,<5
-  - rust
   - scipy
-  - numpy
+  - numpy>=1.22.4
   - pandas
   - scikit-learn
   - codecov
@@ -19,6 +18,7 @@ dependencies:
   - tqdm
   - biom-format
   - pytaxonkit
+  - requests
   - pip
   - sourmash_plugin_branchwater
   - pip:

setup.py

@@ -1,22 +1,72 @@
 from setuptools import setup, find_packages
+from setuptools.command.build_ext import build_ext
+from setuptools.command.install import install
+import os
+import sys
+import subprocess
+import shutil
 
 # Import the version number
-from yacht import __version__
+from src.yacht import __version__
+
+# Custom build class to run the C++ compilation step
+class CustomBuildExt(build_ext):
+    def run(self):
+        # Run the custom build process for C++ code
+        if sys.platform.startswith('win'):
+            # Use the Windows batch file to compile C++ code
+            print("Running Windows build script...")
+            try:
+                subprocess.check_call(['cmd.exe', '/c', 'build_windows.bat'])
+            except subprocess.CalledProcessError as e:
+                print(f"Error during Windows compilation: {e.output}")
+                raise e
+        else:
+            # Use the Unix-based shell script to compile C++ code
+            print("Running Unix-based build script...")
+            try:
+                subprocess.check_call(['bash', 'build_unix.sh'], stderr=subprocess.STDOUT)
+            except subprocess.CalledProcessError as e:
+                print(f"Error during Unix compilation: {e}")
+                raise e
+
+        # Move the compiled binary to the correct location for packaging
+        compiled_binary = os.path.join('src', 'yacht', 'run_yacht_train_core')
+        if os.path.exists(compiled_binary):
+            destination = os.path.join(self.build_lib, 'yacht')
+            os.makedirs(destination, exist_ok=True)
+            shutil.move(compiled_binary, destination)
+        else:
+            print("Compiled binary not found after build step.")
+            raise FileNotFoundError("The executable 'run_yacht_train_core' was not generated successfully.")
+
+        # Run the usual build_ext logic (necessary to continue with setuptools)
+        super().run()
+
+class CustomInstall(install):
+    def run(self):
+        self.run_command('build_ext')
+        super().run()
 
 setup(
     name='yacht',
     version=__version__,
     include_package_data=True,
-    packages=find_packages(),
+    packages=find_packages(where='src'),
+    package_dir={'': 'src'},
+    cmdclass={
+        'build_ext': CustomBuildExt,
+        'install': CustomInstall
+    },
     entry_points={
         'console_scripts': [
             'yacht = yacht:main',
         ],
     },
-    python_requires='>=3.6',
+    python_requires='>3.6,<3.12',
     # Add other package metadata here
     author='Koslicki, D., White, S., Ma, C., & Novikov, A.',
     description='YACHT is a mathematically rigorous hypothesis test for the presence or absence of organisms in a metagenomic sample, based on average nucleotide identity (ANI).',
     license='MIT',
     url='https://github.com/KoslickiLab/YACHT'
-)
+)