Update population spec converter

.github/workflows/scan-images.yml

@@ -83,6 +83,7 @@ jobs:
           - duchy/aws-postgres-internal-server
           - panel-exchange/forwarded-storage-daemon
           - prober/measurement-system-prober
+          - data-provider/population-requisition-fulfiller
     steps:
       - name: Get SARIF file path
         id: get-sarif-file-path

build/variables.bzl

@@ -59,6 +59,9 @@ TEST_K8S_SETTINGS = struct(
     db_secret_name = "$(k8s_db_secret_name)",
     mc_config_secret_name = "$(k8s_mc_config_secret_name)",
     grafana_secret_name = "$(k8s_grafana_secret_name)",
+    pdp1_name = "$(pdp1_name)",
+    pdp1_cert_name = "$(pdp1_cert_name)",
+    population_resource_name = "$(population_resource_name)",
 )
 
 GCLOUD_SETTINGS = struct(

src/main/docker/images.bzl

@@ -101,6 +101,11 @@ COMMON_IMAGES = [
         image = "//src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider:synthetic_generator_edp_simulator_runner_image",
         repository = _PREFIX + "/simulator/synthetic-generator-edp",
     ),
+    struct(
+        name = "population_requisition_fulfiller_image",
+        image = "//src/main/kotlin/org/wfanet/measurement/populationdataprovider:population_requisition_fulfiller_daemon_image",
+        repository = _PREFIX + "/data-provider/population-requisition-fulfiller",
+    ),
     struct(
         name = "access_public_api_server_image",
         image = "//src/main/kotlin/org/wfanet/measurement/access/deploy/common/server:public_api_server_image",

src/main/k8s/BUILD.bazel

@@ -112,3 +112,12 @@ cue_library(
         ":spanner",
     ],
 )
+
+cue_library(
+    name = "population_requisition_fulfiller",
+    srcs = ["population_requisition_fulfiller.cue"],
+    deps = [
+        ":base",
+        ":config",
+    ],
+)

src/main/k8s/local/BUILD.bazel

@@ -188,6 +188,33 @@ cue_dump(
     ],
 )
 
+cue_library(
+    name = "population_requisition_fulfillers_cue",
+    srcs = ["population_requisition_fulfillers.cue"],
+    deps = [
+        ":config_cue",
+        "//src/main/k8s:population_requisition_fulfiller",
+    ],
+)
+
+cue_dump(
+    name = "population_requisition_fulfillers",
+    cue_tags = {
+        "secret_name": SECRET_NAME,
+        "container_registry": IMAGE_REPOSITORY_SETTINGS.container_registry,
+        "image_repo_prefix": IMAGE_REPOSITORY_SETTINGS.repository_prefix,
+        "image_tag": IMAGE_REPOSITORY_SETTINGS.image_tag,
+        "mc_name": TEST_K8S_SETTINGS.mc_name,
+        "pdp1_name": TEST_K8S_SETTINGS.pdp1_name,
+        "pdp1_cert_name": TEST_K8S_SETTINGS.pdp1_cert_name,
+        "population_resource_name": TEST_K8S_SETTINGS.population_resource_name,
+    },
+    tags = ["manual"],
+    deps = [
+        ":population_requisition_fulfillers_cue",
+    ],
+)
+
 cue_library(
     name = "measurement_system_prober_cue",
     srcs = ["measurement_system_prober.cue"],
@@ -386,6 +413,7 @@ kustomization_dir(
         ":emulators",
         ":kingdom",
         ":measurement_system_prober",
+        ":population_requisition_fulfillers",
         ":postgres_database",
     ],
     generate_kustomization = True,

src/main/k8s/local/population_requisition_fulfillers.cue

@@ -0,0 +1,53 @@
+// Copyright 2024 The Cross-Media Measurement Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package k8s
+
+_mc_name:                  string @tag("mc_name")
+_pdp1_name:                string @tag("pdp1_name")
+_pdp1_cert_name:           string @tag("pdp1_cert_name")
+_population_resource_name: string @tag("population_resource_name")
+_secret_name:              string @tag("secret_name")
+
+
+_populationSpec:           "/etc/\(#AppName)/config-files/population_spec_small.textproto"
+_populationRequisitionFulfillerConfigs: [...#PopulationRequisitionFulfillerConfig]
+_populationRequisitionFulfillerConfigs: [
+    {
+        dataProviderDisplayName: "pdp1"
+        dataProviderResourceName: _pdp1_name
+        dataProviderCertResourceName: _pdp1_cert_name
+        populationKeyAndInfoList: [
+            {
+                populationResourceName: _population_resource_name
+                populationSpecFile: _populationSpec
+                eventMessageTypeUrl: "type.googleapis.com/wfa.measurement.api.v2alpha.event_templates.testing.TestEvent"
+            }
+        ]
+        eventMessageDescriptorSet: ["/etc/\(#AppName)/config-files/person_type_set.pb", "/etc/\(#AppName)/config-files/test_event_message_descriptor_set.pb"]
+    }
+]
+
+objectSets: [ for fulfiller in populationRequisitionFulfillers {[fulfiller.deployment]}] +
+	[ for fulfiller in populationRequisitionFulfillers {fulfiller.networkPolicies}]
+
+populationRequisitionFulfillers: {
+    for config in _populationRequisitionFulfillerConfigs {
+        "\(config.dataProviderDisplayName)": #PopulationRequisitionFulfiller & {
+            _imageConfig: repoSuffix: "data-provider/population-requisition-fulfiller"
+            _populationRequisitionFulfillerSecretName: _secret_name
+            _config: config
+        }
+    }
+}

src/main/k8s/population_requisition_fulfiller.cue

@@ -0,0 +1,104 @@
+// Copyright 2024 The Cross-Media Measurement Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package k8s
+
+import "list"
+
+
+#KingdomPublicApiTarget: (#Target & {name: "v2alpha-public-api-server"}).target
+
+#PopulationRequisitionFulfillerConfig: {
+    dataProviderDisplayName:     string
+    dataProviderResourceName:    string
+    dataProviderCertResourceName: string
+    throttlerMinimumInterval:     string | *"2s"
+    eventMessageDescriptorSet:   [...string]
+    populationKeyAndInfoList: [...#PopulationKeyAndInfo]
+}
+
+#PopulationKeyAndInfo: {
+    populationResourceName: string
+    populationSpecFile:     string
+    eventMessageTypeUrl:    string
+}
+
+#PopulationRequisitionFulfiller: {
+    _config: #PopulationRequisitionFulfillerConfig
+    _imageConfig: #ImageConfig
+    _populationRequisitionFulfillerSecretName: string
+
+    let displayName = _config.dataProviderDisplayName
+
+    _populationFlags: {
+    	let flagLists = [ for config in _config.populationKeyAndInfoList {[
+    		"--population-resource-name=\(config.populationResourceName)",
+            "--population-spec=\(config.populationSpecFile)",
+            "--event-message-type-url=\(config.eventMessageTypeUrl)",
+   		]}]
+   		list.FlattenN(flagLists, 2)
+    }
+
+    _eventDescriptorFlags: {
+        let flagLists = [ for file in _config.eventMessageDescriptorSet {[
+            "--event-message-descriptor-set=\(file)"
+        ]}]
+        list.FlattenN(flagLists, 2)
+    }
+
+    deployment: #Deployment & {
+    	_name:       displayName + "-simulator"
+    	_secretName: _populationRequisitionFulfillerSecretName
+    	_system:     "population"
+    	_container: {
+   			image: _imageConfig.image
+            args: [
+                "--kingdom-public-api-target=\(#KingdomPublicApiTarget)",
+                "--kingdom-public-api-cert-host=localhost",
+                "--data-provider-resource-name=\(_config.dataProviderResourceName)",
+                "--data-provider-display-name=\(displayName)",
+                "--data-provider-certificate-resource-name=\(_config.dataProviderCertResourceName)",
+                "--data-provider-encryption-private-keyset=/var/run/secrets/files/\(displayName)_enc_private.tink",
+                "--data-provider-consent-signaling-private-key-der-file=/var/run/secrets/files/\(displayName)_cs_private.der",
+                "--data-provider-consent-signaling-certificate-der-file=/var/run/secrets/files/\(displayName)_cs_cert.der",
+                "--throttler-minimum-interval=\(_config.throttlerMinimumInterval)",
+                "--tls-cert-file=/var/run/secrets/files/\(displayName)_root.pem",
+                "--tls-key-file=/var/run/secrets/files/\(displayName)_root.key",
+   				"--cert-collection-file=/var/run/secrets/files/all_root_certs.pem",
+            ] + _populationFlags + _eventDescriptorFlags
+    	}
+    	spec: template: spec: {
+            _dependencies: [
+				"v2alpha-public-api-server",
+				"worker1-requisition-fulfillment-server",
+                "worker2-requisition-fulfillment-server",
+			]
+            _mounts: "config-files": #ConfigMapMount
+        }
+    }
+
+    networkPolicies: [Name=_]: #NetworkPolicy & {
+        _name: Name
+    }
+
+    networkPolicies: {
+        "\(deployment._name)": {
+            _app_label: deployment.spec.template.metadata.labels.app
+       		_egresses: {
+        	// Need to be able to access Kingdom.
+       			any: {}
+    		}
+        }
+    }
+}

src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/BUILD.bazel

@@ -249,5 +249,7 @@ kt_jvm_library(
     deps = [
         "//src/main/proto/wfa/measurement/api/v2alpha:population_spec_kt_jvm_proto",
         "//src/main/proto/wfa/measurement/api/v2alpha/event_group_metadata/testing:simulator_synthetic_data_spec_kt_jvm_proto",
+        "//src/main/proto/wfa/measurement/api/v2alpha/event_templates/testing:test_event_kt_jvm_proto",
+        "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/common",
     ],
 )

src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/PopulationSpecConverter.kt

@@ -20,7 +20,10 @@ import org.wfanet.measurement.api.v2alpha.PopulationSpec
 import org.wfanet.measurement.api.v2alpha.PopulationSpecKt.subPopulation
 import org.wfanet.measurement.api.v2alpha.PopulationSpecKt.vidRange
 import org.wfanet.measurement.api.v2alpha.event_group_metadata.testing.SyntheticPopulationSpec
+import org.wfanet.measurement.api.v2alpha.event_templates.testing.Person
+import org.wfanet.measurement.api.v2alpha.event_templates.testing.person
 import org.wfanet.measurement.api.v2alpha.populationSpec
+import org.wfanet.measurement.common.pack
 
 fun SyntheticPopulationSpec.toPopulationSpec(): PopulationSpec {
   return populationSpec {
@@ -31,6 +34,11 @@ fun SyntheticPopulationSpec.toPopulationSpec(): PopulationSpec {
             startVid = it.vidSubRange.start
             endVidInclusive = (it.vidSubRange.endExclusive - 1)
           }
+          attributes += person {
+            gender = Person.Gender.forNumber(it.populationFieldsValuesMap["person.gender"]!!.enumValue)
+            ageGroup = Person.AgeGroup.forNumber(it.populationFieldsValuesMap["person.age_group"]!!.enumValue)
+            socialGradeGroup = Person.SocialGradeGroup.forNumber(it.populationFieldsValuesMap["person.social_grade_group"]!!.enumValue)
+          }.pack()
         }
       }
   }

src/main/kotlin/org/wfanet/measurement/populationdataprovider/BUILD.bazel

@@ -1,4 +1,6 @@
+load("@rules_java//java:defs.bzl", "java_binary")
 load("@wfa_rules_kotlin_jvm//kotlin:defs.bzl", "kt_jvm_library")
+load("//src/main/docker:macros.bzl", "java_image")
 
 package(
     default_visibility = [
@@ -44,3 +46,19 @@ kt_jvm_library(
         "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/storage:client",
     ],
 )
+
+java_binary(
+    name = "PopulationRequisitionFulfillerDaemon",
+    main_class = "org.wfanet.measurement.populationdataprovider.PopulationRequisitionFulfillerDaemonKt",
+    runtime_deps = [
+        ":population_requisition_fulfiller_daemon",
+        "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/gcloud/logging",
+    ],
+)
+
+java_image(
+    name = "population_requisition_fulfiller_daemon_image",
+    binary = ":PopulationRequisitionFulfillerDaemon",
+    main_class = "org.wfanet.measurement.populationdataprovider.PopulationRequisitionFulfillerDaemonKt",
+    visibility = ["//src:docker_image_deployment"],
+)