Skip to content

Commit

Permalink
mill update
Browse files Browse the repository at this point in the history
  • Loading branch information
patrick-schultz committed Dec 17, 2024
1 parent 6dee8a2 commit 8c5873b
Show file tree
Hide file tree
Showing 13 changed files with 139 additions and 317 deletions.
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ pylint-hailtop:

.PHONY: check-hail
check-hail: check-hail-fast pylint-hailtop
cd hail && sh millw __.checkFormat + __.fix --check
cd hail && sh mill __.checkFormat + __.fix --check

.PHONY: check-services
check-services: $(CHECK_SERVICES_MODULES)
Expand Down
8 changes: 4 additions & 4 deletions build.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -724,7 +724,7 @@ steps:
cd /io/repo/hail
export MILLOPTS='--no-server' HAIL_RELEASE_MODE=1
time retry sh millw --no-server --version
time retry sh mill --no-server --version
# We've encountered the following sporadic error in CI between `mill`
# finishing assembling the jar and `Make` copying the jar:
Expand Down Expand Up @@ -770,7 +770,7 @@ steps:
cd /io/repo/hail
export MILLOPTS='--no-server'
time retry sh millw --no-server --version
time retry sh mill --no-server --version
# See `build_hail_jar_and_wheel`
time retry make shadowJar
Expand Down Expand Up @@ -803,7 +803,7 @@ steps:
cd /io/repo/hail
export MILLOPTS='--no-server'
time retry sh millw --no-server --version
time retry sh mill --no-server --version
# See `build_hail_jar_and_wheel`
time retry make shadowTestJar
Expand Down Expand Up @@ -863,7 +863,7 @@ steps:
cd /io/repo/hail
export SPARK_VERSION="3.0.2" SCALA_VERSION="2.12.13" HAIL_RELEASE_MODE=1 MILLOPTS='--no-server'
time retry sh millw --no-server --version
time retry sh mill --no-server --version
# See `build_hail_jar_and_wheel`
time retry make shadowJar
Expand Down
2 changes: 1 addition & 1 deletion hail/.mill-version
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0.11.7
0.12.4
2 changes: 1 addition & 1 deletion hail/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ CLOUD_HAIL_DOCTEST_DATA_DIR = $(CLOUD_HAIL_TEST_RESOURCES_PREFIX)/doctest/data/

# mill integration

MILL := bash millw
MILL := bash mill
MILLOPTS ?=

.PHONY: mill-clean
Expand Down
122 changes: 59 additions & 63 deletions hail/build.sc → hail/build.mill
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import $ivy.`com.goyeau::mill-scalafix::0.3.2`
import $ivy.`de.tototec::de.tobiasroeser.mill.vcs.version::0.4.0`
package build

import $ivy.`com.goyeau::mill-scalafix::0.4.2`
import $ivy.`de.tototec::de.tobiasroeser.mill.vcs.version::0.4.1`
import com.goyeau.mill.scalafix.ScalafixModule
import de.tobiasroeser.mill.vcs.version.VcsVersion
import mill._
Expand All @@ -15,56 +17,6 @@ object Settings {
val hailPatchVersion = "133"
}

/** Update the millw script. */
def millw(): Command[PathRef] = T.command {
val target =
mill.util.Util.download("https://raw.githubusercontent.com/lefou/millw/main/millw")
val millw = T.workspace / "millw"
os.copy.over(target.path, millw)
os.perms.set(millw, os.perms(millw) + java.nio.file.attribute.PosixFilePermission.OWNER_EXECUTE)
target
}

def scalaVersion: T[String] = T.input {
val v = T.ctx().env.getOrElse("SCALA_VERSION", "2.12.15")
if (!v.startsWith("2.12"))
Result.Failure("Hail currently supports only Scala 2.12")
else
v
}

def javaVersion: T[String] = T.input {
System.getProperty("java.version")
}

def sparkVersion: T[String] = T.input {
Result.Success(T.ctx().env.getOrElse("SPARK_VERSION", "3.5.0"))
}

def debugMode: T[Boolean] = T.input {
val isDebug = !T.ctx().env.contains("HAIL_RELEASE_MODE")
T.log.info(s"Building in ${if (isDebug) "debug" else "release"} mode")
isDebug
}

def debugOrRelease: Task[String] = T.task {
if (debugMode()) "debug" else "release"
}

def buildInfo: T[PathRef] = T {
val revision = VcsVersion.vcsState().currentRevision
os.write(
T.dest / "build-info.properties",
s"""[Build Metadata]
|revision=$revision
|sparkVersion=${sparkVersion()}
|hailPipVersion=${Settings.hailMajorMinorVersion}.${Settings.hailPatchVersion}
|hailBuildConfiguration=${debugOrRelease()}
|""".stripMargin,
)
PathRef(T.dest)
}

object Deps {
object HTTPComponents {
val core = ivy"org.apache.httpcomponents:httpcore:4.4.14"
Expand Down Expand Up @@ -94,8 +46,8 @@ object Deps {
}

object Spark {
def core: Task[Dep] = T.task(ivy"org.apache.spark::spark-core:${sparkVersion()}")
def mllib: Task[Dep] = T.task(ivy"org.apache.spark::spark-mllib:${sparkVersion()}")
def core: Task[Dep] = Task.Anon(ivy"org.apache.spark::spark-core:${build.env.sparkVersion()}")
def mllib: Task[Dep] = Task.Anon(ivy"org.apache.spark::spark-mllib:${build.env.sparkVersion()}")
}

val samtools = ivy"com.github.samtools:htsjdk:3.0.5"
Expand All @@ -121,13 +73,13 @@ object Deps {
}

trait HailScalaModule extends SbtModule with ScalafmtModule with ScalafixModule { outer =>
override def scalaVersion: T[String] = build.scalaVersion()
override def scalaVersion: T[String] = build.env.scalaVersion()

override def javacOptions: T[Seq[String]] = Seq(
"-Xlint:all",
"-Werror",
if (debugMode()) "-g" else "-O",
) ++ (if (!javaVersion().startsWith("1.8")) Seq("-Xlint:-processing") else Seq())
if (build.env.debugMode()) "-g" else "-O",
) ++ (if (!build.env.javaVersion().startsWith("1.8")) Seq("-Xlint:-processing") else Seq())

override def scalacOptions: T[Seq[String]] = T {
Seq(
Expand All @@ -139,8 +91,9 @@ trait HailScalaModule extends SbtModule with ScalafmtModule with ScalafixModule
"-Yno-adapted-args", // will be removed in 2.13
"-Xlint",
"-Ywarn-unused:_,-explicits,-implicits",
"-Wconf:msg=legacy-binding:s",
) ++ (
if (debugMode()) Seq()
if (build.env.debugMode()) Seq()
else Seq(
"-Xfatal-warnings",
"-opt:l:method",
Expand All @@ -153,7 +106,7 @@ trait HailScalaModule extends SbtModule with ScalafmtModule with ScalafixModule
override def bspCompileClasspath: T[Agg[UnresolvedPath]] =
super.bspCompileClasspath() ++ resources().map(p => UnresolvedPath.ResolvedPath(p.path))

trait HailTests extends SbtModuleTests with TestNg with ScalafmtModule with ScalafixModule {
trait HailTests extends SbtTests with TestNg with ScalafmtModule with ScalafixModule {
override def forkArgs: T[Seq[String]] = Seq("-Xss4m", "-Xmx4096M")

override def ivyDeps: T[Agg[Dep]] =
Expand All @@ -171,7 +124,50 @@ trait HailScalaModule extends SbtModule with ScalafmtModule with ScalafixModule
}
}

object main extends RootModule with HailScalaModule { outer =>
object `package` extends RootModule with HailScalaModule { outer =>

object env extends Module {
def scalaVersion: T[String] = Task.Input {
val v = T.ctx().env.getOrElse("SCALA_VERSION", "2.12.20")
if (!v.startsWith("2.12"))
Result.Failure("Hail currently supports only Scala 2.12")
else
v
}

def javaVersion: T[String] = Task.Input {
System.getProperty("java.version")
}

def sparkVersion: T[String] = Task.Input {
Result.Success(T.ctx().env.getOrElse("SPARK_VERSION", "3.5.0"))
}

def debugMode: T[Boolean] = Task.Input {
val isDebug = !T.ctx().env.contains("HAIL_RELEASE_MODE")
T.log.info(s"Building in ${if (isDebug) "debug" else "release"} mode")
isDebug
}

def debugOrRelease: Task[String] = Task.Anon {
if (debugMode()) "debug" else "release"
}
}

def buildInfo: T[PathRef] = T {
val revision = VcsVersion.vcsState().currentRevision
os.write(
T.dest / "build-info.properties",
s"""[Build Metadata]
|revision=$revision
|sparkVersion=${env.sparkVersion()}
|hailPipVersion=${Settings.hailMajorMinorVersion}.${Settings.hailPatchVersion}
|hailBuildConfiguration=${env.debugOrRelease()}
|""".stripMargin,
)
PathRef(T.dest)
}

override def moduleDeps: Seq[JavaModule] = Seq(memory)

override def resources: T[Seq[PathRef]] = Seq(
Expand Down Expand Up @@ -256,15 +252,15 @@ object main extends RootModule with HailScalaModule { outer =>

override def javacOptions: T[Seq[String]] =
outer.javacOptions() ++ (
if (javaVersion().startsWith("1.8")) Seq(
if (env.javaVersion().startsWith("1.8")) Seq(
"-XDenableSunApiLintControl",
"-Xlint:-sunapi",
)
else Seq()
)

override def sources: T[Seq[PathRef]] = T.sources {
Seq(PathRef(this.millSourcePath / os.up / "src" / debugOrRelease() / "java"))
override def sources: T[Seq[PathRef]] = Task.Sources {
Seq(PathRef(this.millSourcePath / os.up / "src" / env.debugOrRelease() / "java"))
}
}

Expand Down
67 changes: 67 additions & 0 deletions hail/mill
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
#!/usr/bin/env sh

# This is a wrapper script, that automatically download mill from GitHub release pages
# You can give the required mill version with MILL_VERSION env variable
# If no version is given, it falls back to the value of DEFAULT_MILL_VERSION

set -e

if [ -z "${DEFAULT_MILL_VERSION}" ] ; then
DEFAULT_MILL_VERSION=0.12.3
fi

if [ -z "$MILL_VERSION" ] ; then
if [ -f ".mill-version" ] ; then
MILL_VERSION="$(tr '\r' '\n' < .mill-version | head -n 1 2> /dev/null)"
elif [ -f ".config/mill-version" ] ; then
MILL_VERSION="$(tr '\r' '\n' < .config/mill-version | head -n 1 2> /dev/null)"
elif [ -f "mill" ] && [ "$0" != "mill" ] ; then
MILL_VERSION=$(grep -F "DEFAULT_MILL_VERSION=" "mill" | head -n 1 | cut -d= -f2)
else
MILL_VERSION=$DEFAULT_MILL_VERSION
fi
fi

if [ "x${XDG_CACHE_HOME}" != "x" ] ; then
MILL_DOWNLOAD_PATH="${XDG_CACHE_HOME}/mill/download"
else
MILL_DOWNLOAD_PATH="${HOME}/.cache/mill/download"
fi
MILL_EXEC_PATH="${MILL_DOWNLOAD_PATH}/${MILL_VERSION}"

version_remainder="$MILL_VERSION"
MILL_MAJOR_VERSION="${version_remainder%%.*}"; version_remainder="${version_remainder#*.}"
MILL_MINOR_VERSION="${version_remainder%%.*}"; version_remainder="${version_remainder#*.}"

if [ ! -s "$MILL_EXEC_PATH" ] ; then
mkdir -p "$MILL_DOWNLOAD_PATH"
if [ "$MILL_MAJOR_VERSION" -gt 0 ] || [ "$MILL_MINOR_VERSION" -ge 5 ] ; then
ASSEMBLY="-assembly"
fi
DOWNLOAD_FILE=$MILL_EXEC_PATH-tmp-download
MILL_VERSION_TAG=$(echo $MILL_VERSION | sed -E 's/([^-]+)(-M[0-9]+)?(-.*)?/\1\2/')
MILL_DOWNLOAD_URL="https://repo1.maven.org/maven2/com/lihaoyi/mill-dist/$MILL_VERSION/mill-dist-$MILL_VERSION.jar"
curl --fail -L -o "$DOWNLOAD_FILE" "$MILL_DOWNLOAD_URL"
chmod +x "$DOWNLOAD_FILE"
mv "$DOWNLOAD_FILE" "$MILL_EXEC_PATH"
unset DOWNLOAD_FILE
unset MILL_DOWNLOAD_URL
fi

if [ -z "$MILL_MAIN_CLI" ] ; then
MILL_MAIN_CLI="${0}"
fi

MILL_FIRST_ARG=""

# first arg is a long flag for "--interactive" or starts with "-i"
if [ "$1" = "--bsp" ] || [ "${1#"-i"}" != "$1" ] || [ "$1" = "--interactive" ] || [ "$1" = "--no-server" ] || [ "$1" = "--repl" ] || [ "$1" = "--help" ] ; then
# Need to preserve the first position of those listed options
MILL_FIRST_ARG=$1
shift
fi

unset MILL_DOWNLOAD_PATH
unset MILL_VERSION

exec $MILL_EXEC_PATH $MILL_FIRST_ARG -D "mill.main.cli=${MILL_MAIN_CLI}" "$@"
Loading

0 comments on commit 8c5873b

Please sign in to comment.