Merge pull request #202 from sourcerer-io/develop

feat: release new processing flow

build.gradle

@@ -36,8 +36,8 @@ buildConfig {
     buildConfigField 'String', 'PROFILE_URL', 'https://sourcerer.io/'
 
     // App version.
-    buildConfigField 'int', 'VERSION_CODE', '6'
-    buildConfigField 'String', 'VERSION', '0.2.1'
+    buildConfigField 'int', 'VERSION_CODE', '7'
+    buildConfigField 'String', 'VERSION', '0.2.2'
 
     // Logging.
     buildConfigField 'String', 'ENV', project.hasProperty('env') ? env : 'production'
@@ -48,7 +48,7 @@ buildConfig {
     buildConfigField 'boolean', 'PRINT_STACK_TRACE', 'false'
 
     // Google Analytics.
-    buildConfigField 'String', 'GA_BASE_PATH', 'http://www.google-analytics.com'
+    buildConfigField 'String', 'GA_BASE_PATH', 'https://www.google-analytics.com'
     buildConfigField 'String', 'GA_TRACKING_ID', 'UA-107129190-2'
     buildConfigField 'boolean', 'IS_GA_ENABLED', 'true'
 
@@ -59,6 +59,7 @@ buildConfig {
     buildConfigField 'boolean', 'COMMIT_HASHER_ENABLED', project.hasProperty('commit-hasher-enabled') ? project.property('commit-hasher-enabled').toString() : 'true'
     buildConfigField 'boolean', 'FACT_HASHER_ENABLED', project.hasProperty('fact-hasher-enabled') ? project.property('fact-hasher-enabled').toString() : 'true'
     buildConfigField 'boolean', 'LONGEVITY_ENABLED', project.hasProperty('longevity-enabled') ? project.property('longevity-enabled').toString() : 'false'
+    buildConfigField 'long', 'HEARTBEAT_RATE', project.hasProperty('heartbeat-rate') ? project.property('heartbeat-rate').toString() : '60000'
 
     buildConfig
 }

src/main/kotlin/app/Analytics.kt

@@ -72,8 +72,11 @@ object Analytics {
 
         try {
             // Send event to GA with united params.
-            post(params + defaultParams.filter { !params.contains(it) }
-                + idParams).responseString()
+            val (_, _, result) = post(params +
+                defaultParams.filter { !params.contains(it) } +
+                idParams).responseString()
+            val (_, e) = result
+            if (e != null) { throw e }
         } catch (e: Throwable) {
             Logger.error(e, "Error while sending GA report", logOnly = true)
         }

src/main/kotlin/app/api/Api.kt

@@ -6,21 +6,31 @@ package app.api
 import app.model.Author
 import app.model.Commit
 import app.model.Fact
+import app.model.Process
+import app.model.ProcessEntry
 import app.model.Repo
 import app.model.User
 
 interface Api {
     companion object {
         val OUT_OF_DATE = 1
+        val PROCESS_STATUS_START = 100
+        val PROCESS_STATUS_COMPLETE = 200
+        val PROCESS_STATUS_FAIL = 1000
+        val PROCESS_ERROR_RESTRICTED = 2
+        val PROCESS_ERROR_TOO_BIG = 3
+        val PROCESS_ERROR_TOO_MUCH_COMMITS = 4
+        val PROCESS_ERROR_NO_COMMITS = 5
     }
 
     fun authorize(): Result<Unit>
     fun getUser(): Result<User>
     fun postUser(user: User): Result<Unit>
     fun postRepo(repo: Repo): Result<Repo>
-    fun postComplete(): Result<Unit>
     fun postCommits(commitsList: List<Commit>): Result<Unit>
     fun deleteCommits(commitsList: List<Commit>): Result<Unit>
     fun postFacts(factsList: List<Fact>): Result<Unit>
     fun postAuthors(authorsList: List<Author>): Result<Unit>
+    fun postProcessCreate(requestNumEntries: Int): Result<Process>
+    fun postProcess(processEntries: List<ProcessEntry>): Result<Unit>
 }

src/main/kotlin/app/api/MockApi.kt

@@ -8,19 +8,23 @@ import app.model.Author
 import app.model.Commit
 import app.model.Repo
 import app.model.Fact
+import app.model.Process
+import app.model.ProcessEntry
 import app.model.User
 
 class MockApi(  // GET requests.
     var mockUser: User = User(),
-    var mockRepo: Repo = Repo()) : Api {
+    var mockRepo: Repo = Repo(),
+    var mockProcessEntries: List<ProcessEntry> = listOf()) : Api {
     // POST requests.
     // In case of multiple requests.
     var receivedRepos: MutableList<Repo> = mutableListOf()
     var receivedAddedCommits: MutableList<Commit> = mutableListOf()
     var receivedFacts: MutableList<Fact> = mutableListOf()
     var receivedAuthors: MutableList<Author> = mutableListOf()
     var receivedUsers: MutableList<User> = mutableListOf()
-    var receivedComplete: Int = 0
+    var receivedProcessCreate: MutableList<Process> = mutableListOf()
+    var receivedProcess: MutableList<Process> = mutableListOf()
 
     // DELETE requests.
     var receivedDeletedCommits: MutableList<Commit> = mutableListOf()
@@ -47,12 +51,6 @@ class MockApi(  // GET requests.
         return Result(mockRepo)
     }
 
-    override fun postComplete(): Result<Unit> {
-        Logger.debug { "MockApi: postComplete request " }
-        receivedComplete++
-        return Result()
-    }
-
     override fun postCommits(commitsList: List<Commit>): Result<Unit> {
         Logger.debug {
             "MockApi: postCommits request (${commitsList.size} commits)"
@@ -80,4 +78,19 @@ class MockApi(  // GET requests.
         receivedAuthors.addAll(authorsList)
         return Result()
     }
+
+    override fun postProcessCreate(requestNumEntries: Int): Result<Process> {
+        Logger.debug { "MockApi: postProcessCreate request " +
+            "($requestNumEntries entries requested)" }
+        receivedProcessCreate.add(
+            Process(requestNumEntries = requestNumEntries))
+        return Result(Process(entries = mockProcessEntries))
+    }
+
+    override fun postProcess(processEntries: List<ProcessEntry>): Result<Unit> {
+        Logger.debug { "MockApi: postProcess request (${processEntries.size} " +
+            "entries updated)" }
+        receivedProcess.add(Process(entries = processEntries))
+        return Result()
+    }
 }

src/main/kotlin/app/api/ServerApi.kt

@@ -12,6 +12,8 @@ import app.model.Commit
 import app.model.CommitGroup
 import app.model.Fact
 import app.model.FactGroup
+import app.model.Process
+import app.model.ProcessEntry
 import app.model.Repo
 import app.model.User
 import com.github.kittinunf.fuel.core.FuelManager
@@ -93,10 +95,6 @@ class ServerApi (private val configurator: Configurator) : Api {
                             .body(repo.serialize())
     }
 
-    private fun createRequestPostComplete(): Request {
-        return post("/complete").header(getContentTypeHeader())
-    }
-
     private fun createRequestPostCommits(commits: CommitGroup): Request {
         return post("/commits").header(getContentTypeHeader())
                                .body(commits.serialize())
@@ -117,6 +115,16 @@ class ServerApi (private val configurator: Configurator) : Api {
                                .body(authors.serialize())
     }
 
+    private fun createRequestPostProcessCreate(process: Process): Request {
+        return post("/process/create").header(getContentTypeHeader())
+                                      .body(process.serialize())
+    }
+
+    private fun createRequestPostProcess(process: Process): Request {
+        return post("/process").header(getContentTypeHeader())
+                               .body(process.serialize())
+    }
+
     private fun <T> makeRequest(request: Request,
                                 requestName: String,
                                 parser: (ByteArray) -> T): Result<T> {
@@ -172,11 +180,6 @@ class ServerApi (private val configurator: Configurator) : Api {
                            { body -> Repo(body) })
     }
 
-    override fun postComplete(): Result<Unit> {
-        return makeRequest(createRequestPostComplete(),
-                           "postComplete", {})
-    }
-
     override fun postCommits(commitsList: List<Commit>): Result<Unit> {
         val commits = CommitGroup(commitsList)
         return makeRequest(createRequestPostCommits(commits),
@@ -198,4 +201,17 @@ class ServerApi (private val configurator: Configurator) : Api {
         val authors = AuthorGroup(authorsList)
         return makeRequest(createRequestPostAuthors(authors), "postAuthors", {})
     }
+
+    override fun postProcessCreate(requestNumEntries: Int): Result<Process> {
+        val process = Process(requestNumEntries = requestNumEntries)
+        return makeRequest(createRequestPostProcessCreate(process),
+                           "postProcessCreate", { body -> Process(body) })
+    }
+
+    override fun postProcess(processEntries: List<ProcessEntry>): Result<Unit> {
+        // TODO(anatoly): Restrict possible status and error codes on CS.
+        val process = Process(entries = processEntries)
+        return makeRequest(createRequestPostProcess(process), "postProcess", {})
+    }
+
 }

src/main/kotlin/app/extractors/CommonExtractor.kt

@@ -29,7 +29,6 @@ class CommonExtractor : ExtractorInterface {
             reversedMap.put("html", listOf("html", "htm"))
             reversedMap.put("j", listOf("ijs"))
             reversedMap.put("julia", listOf("jl"))
-            reversedMap.put("kotlin", listOf("kt"))
             reversedMap.put("lisp", listOf("lisp", "lsp", "l"))
             reversedMap.put("lua", listOf("lua"))
             reversedMap.put("makefile", listOf("makefile"))
@@ -74,6 +73,7 @@ class CommonExtractor : ExtractorInterface {
                 file
             } else null
         }
+
         return super.extract(files)
     }
 }

src/main/kotlin/app/extractors/CssExtractor.kt

@@ -0,0 +1,35 @@
+// Copyright 2018 Sourcerer Inc. All Rights Reserved.
+// Author: Liubov Yaronskaya (lyaronskaya@sourcerer.io)
+
+package app.extractors
+
+import app.model.CommitStats
+import app.model.DiffFile
+
+class CssExtractor : ExtractorInterface {
+    companion object {
+        val LANGUAGE_NAME = "css"
+        val FILE_EXTS = listOf("css", "scss", "less", "sass")
+    }
+
+    override fun extract(files: List<DiffFile>): List<CommitStats> {
+        files.map { file -> file.language = LANGUAGE_NAME }
+        val stats = FILE_EXTS.filter { it != "css" }.map { extension ->
+            val result = files.filter { it.extension == extension }
+                .fold(Pair(0, 0)) { total, file ->
+                    val currentNumAdded = file.getAllAdded()
+                                              .filter { it.isNotBlank() }.size
+                    val currentNumDeleted = file.getAllDeleted()
+                                                .filter { it.isNotBlank() }.size
+                    Pair(total.first + currentNumAdded,
+                         total.second + currentNumDeleted)}.toList()
+
+            CommitStats(numLinesAdded = result[0],
+                        numLinesDeleted = result[1],
+                        type = Extractor.TYPE_LIBRARY,
+                        tech = extension)
+        }.filter { it.numLinesAdded > 0 || it.numLinesDeleted > 0 }
+
+        return stats + super.extract(files)
+    }
+}

src/main/kotlin/app/extractors/Extractor.kt

@@ -31,6 +31,7 @@ class Extractor : ExtractorInterface {
             in ObjectiveCExtractor.FILE_EXTS -> ObjectiveCExtractor()
             in SwiftExtractor.FILE_EXTS -> SwiftExtractor()
             in KotlinExtractor.FILE_EXTS -> KotlinExtractor()
+            in CssExtractor.FILE_EXTS -> CssExtractor()
             else -> CommonExtractor()
         }
     }

src/main/kotlin/app/hashers/RepoHasher.kt

@@ -9,6 +9,7 @@ import app.api.Api
 import app.config.Configurator
 import app.model.Author
 import app.model.LocalRepo
+import app.model.ProcessEntry
 import app.model.Repo
 import app.utils.FileHelper.toPath
 import app.utils.HashingException
@@ -19,7 +20,8 @@ import java.io.IOException
 import kotlin.collections.HashSet
 
 class RepoHasher(private val localRepo: LocalRepo, private val api: Api,
-                 private val configurator: Configurator) {
+                 private val configurator: Configurator,
+                 private val processEntryId: Int? = null) {
     var serverRepo: Repo = Repo()
 
     init {
@@ -32,6 +34,7 @@ class RepoHasher(private val localRepo: LocalRepo, private val api: Api,
         Logger.info { "Hashing of repo started" }
         val git = loadGit(localRepo.path)
         try {
+            updateProcess(processEntryId, Api.PROCESS_STATUS_START)
             val (rehashes, authors) = CommitCrawler.fetchRehashesAndAuthors(git)
 
             localRepo.parseGitConfig(git.repository.config)
@@ -88,11 +91,13 @@ class RepoHasher(private val localRepo: LocalRepo, private val api: Api,
             if (errors.isNotEmpty()) {
                 throw HashingException(errors)
             }
-
             Logger.info(Logger.Events.HASHING_REPO_SUCCESS)
                 { "Hashing repo completed" }
-        }
-        finally {
+            updateProcess(processEntryId, Api.PROCESS_STATUS_COMPLETE)
+        } catch (e: Throwable) {
+            updateProcess(processEntryId, Api.PROCESS_STATUS_FAIL)
+            throw e
+        } finally {
             closeGit(git)
         }
     }
@@ -143,4 +148,15 @@ class RepoHasher(private val localRepo: LocalRepo, private val api: Api,
 
         return knownEmails.filter { emails.contains(it) }.toHashSet()
     }
+
+    private fun updateProcess(processEntryId: Int?, status: Int,
+                              errorCode: Int = 0) {
+        if (processEntryId == null) {
+            return
+        }
+
+        val processEntry = ProcessEntry(id = processEntryId, status = status,
+            errorCode = errorCode)
+        api.postProcess(listOf(processEntry)).onErrorThrow()
+    }
 }

src/main/kotlin/app/model/Process.kt

@@ -0,0 +1,38 @@
+package app.model
+
+import app.Protos
+import com.google.protobuf.InvalidProtocolBufferException
+import java.security.InvalidParameterException
+
+/**
+ * Used to describe processing of multiple repos.
+ */
+data class Process(
+    var id: Int = 0,
+    var requestNumEntries: Int = 0,
+    var entries: List<ProcessEntry> = mutableListOf()
+) {
+    @Throws(InvalidParameterException::class)
+    constructor(proto: Protos.Process) : this() {
+        id = proto.id
+        requestNumEntries = proto.requestNumEntries
+        entries = proto.entriesList.map { ProcessEntry(it) }
+    }
+
+    @Throws(InvalidProtocolBufferException::class)
+    constructor(bytes: ByteArray) : this(Protos.Process.parseFrom(bytes))
+
+    constructor(serialized: String) : this(serialized.toByteArray())
+
+    fun getProto(): Protos.Process {
+        return Protos.Process.newBuilder()
+                     .setId(id)
+                     .setRequestNumEntries(requestNumEntries)
+                     .addAllEntries(entries.map { it.getProto() })
+                     .build()
+    }
+
+    fun serialize(): ByteArray {
+        return getProto().toByteArray()
+    }
+}

src/main/kotlin/app/model/ProcessEntry.kt

@@ -0,0 +1,38 @@
+package app.model
+
+import app.Protos
+import com.google.protobuf.InvalidProtocolBufferException
+import java.security.InvalidParameterException
+
+/**
+ * Used to describe processing of a single repo.
+ */
+data class ProcessEntry(
+    var id: Int = 0,
+    var status: Int = 0,
+    var errorCode: Int = 0
+) {
+    @Throws(InvalidParameterException::class)
+    constructor(proto: Protos.ProcessEntry) : this() {
+        id = proto.id
+        status = proto.status
+        errorCode = proto.errorCode
+    }
+
+    @Throws(InvalidProtocolBufferException::class)
+    constructor(bytes: ByteArray) : this(Protos.ProcessEntry.parseFrom(bytes))
+
+    constructor(serialized: String) : this(serialized.toByteArray())
+
+    fun getProto(): Protos.ProcessEntry {
+        return Protos.ProcessEntry.newBuilder()
+                     .setId(id)
+                     .setStatus(status)
+                     .setErrorCode(errorCode)
+                     .build()
+    }
+
+    fun serialize(): ByteArray {
+        return getProto().toByteArray()
+    }
+}