Store to sqlite

Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com>

pyproject.toml

@@ -15,7 +15,7 @@ dependencies = [
     "packageurl-python",
     "cvss",
     "pydantic[email]",
-    "pysqlite3-binary"
+    "pysqlite3"
 ]
 requires-python = ">=3.10"
 readme = "README.md"

vdb/cli.py

@@ -3,7 +3,9 @@
 
 import argparse
 import logging
+import os
 import re
+import shutil
 
 from tabulate import tabulate
 
@@ -160,7 +162,8 @@ def main():
     args = build_args()
     print(AT_LOGO)
     if args.clean:
-        db_lib.clear_all()
+        if os.path.exists(config.DATA_DIR):
+            shutil.rmtree(config.DATA_DIR, ignore_errors=True)
     if args.cache or args.cache_os:
         db_lib.clear_all()
         if args.only_osv:

vdb/lib/cve.py

@@ -1,5 +1,4 @@
 import os
-import pickle
 import uuid
 
 import orjson
@@ -285,23 +284,19 @@ def to_cve_containers(avuln: Vulnerability) -> CnaPublishedContainer | None:
         problem_types = []
         for acwe in avuln.problem_type.split(","):
             problem_types.append(
-                ProblemTypes(
-                    [
-                        ProblemType(
-                            descriptions=[
-                                Description1(
-                                    lang=Language("en"),
-                                    description=acwe,
-                                    cweId=acwe,
-                                    type="CWE",
-                                )
-                            ]
+                ProblemType(
+                    descriptions=[
+                        Description1(
+                            lang=Language("en"),
+                            description=acwe,
+                            cweId=acwe,
+                            type="CWE",
                         )
                     ]
                 )
             )
         if problem_types:
-            cont.problemTypes = problem_types
+            cont.problemTypes = ProblemTypes(problem_types)
     return cont
 
 
@@ -358,11 +353,17 @@ def store5(self, data: list[CVE]):
             if d.containers.cna and d.containers.cna.affected:
                 for affected in d.containers.cna.affected.root:
                     versions: list[Versions] = affected.versions
+                    source_data = d.model_dump(mode="json",
+                                               exclude_defaults=True,
+                                               exclude_unset=True,
+                                               exclude_none=True)
                     self.db_conn.execute(
-                        "INSERT INTO cve_data values(?, ?, ?, ?, ?, ?);", (
+                        "INSERT INTO cve_data values(?, ?, ?, ?, jsonb(?), ?);", (
                             cve_id.model_dump(mode="python"), affected.vendor,
                             affected.product,
-                            affected.packageName, pickle.dumps(d), None))
+                            affected.packageName,
+                            orjson.dumps(source_data).decode("utf-8", "ignore"),
+                            None))
                     cleaned_versions = [v.model_dump(mode="json", exclude_none=True) for v in versions]
                     self.index_conn.execute(
                         "INSERT INTO cve_index values(?, ?, ?, ?, jsonb(?));", (

vdb/lib/cve_model/__init__.py

@@ -87,16 +87,6 @@ class ShortName(RootModel[str]):
     ]
 
 
-class Timestamp(RootModel[AwareDatetime]):
-    root: Annotated[
-        AwareDatetime,
-        Field(
-            description="Date/time format based on RFC3339 and ISO ISO8601, with an optional timezone in the format 'yyyy-MM-ddTHH:mm:ssZZZZ'. If timezone offset is not given, GMT (0000) is assumed.",
-            pattern="^(((2000|2400|2800|(19|2[0-9](0[48]|[2468][048]|[13579][26])))-02-29)|(((19|2[0-9])[0-9]{2})-02-(0[1-9]|1[0-9]|2[0-8]))|(((19|2[0-9])[0-9]{2})-(0[13578]|10|12)-(0[1-9]|[12][0-9]|3[01]))|(((19|2[0-9])[0-9]{2})-(0[469]|11)-(0[1-9]|[12][0-9]|30)))T(2[0-3]|[01][0-9]):([0-5][0-9]):([0-5][0-9])(\\.[0-9]+)?(Z|[+-][0-9]{2}:[0-9]{2})?$",
-        ),
-    ]
-
-
 class Version(RootModel[str]):
     root: Annotated[
         str,
@@ -523,7 +513,7 @@ class CveMetadataPublished(BaseModel):
         Field(None, description="The user that requested the CVE identifier."),
     ]
     dateUpdated: Annotated[
-        Optional[Timestamp],
+        Optional[AwareDatetime],
         Field(None, description="The date/time the record was last updated."),
     ]
     serial: Annotated[
@@ -535,14 +525,14 @@ class CveMetadataPublished(BaseModel):
         ),
     ]
     dateReserved: Annotated[
-        Optional[Timestamp],
+        Optional[AwareDatetime],
         Field(
             None,
             description="The date/time this CVE ID was reserved in the CVE automation workgroup services system. Disclaimer: This date reflects when the CVE ID was reserved, and does not necessarily indicate when this vulnerability was discovered, shared with the affected vendor, publicly disclosed, or updated in CVE.",
         ),
     ]
     datePublished: Annotated[
-        Optional[Timestamp],
+        Optional[AwareDatetime],
         Field(
             None,
             description="The date/time the CVE Record was first published in the CVE List.",
@@ -584,23 +574,23 @@ class CveMetadataRejected(BaseModel):
         ),
     ]
     dateUpdated: Annotated[
-        Optional[Timestamp],
+        Optional[AwareDatetime],
         Field(None, description="The date/time the record was last updated."),
     ]
     datePublished: Annotated[
-        Optional[Timestamp],
+        Optional[AwareDatetime],
         Field(
             None,
             description="The date/time the CVE Record was first published in the CVE List.",
         ),
     ]
     dateRejected: Annotated[
-        Optional[Timestamp],
+        Optional[AwareDatetime],
         Field(None, description="The date/time the CVE ID was rejected."),
     ]
     state: Annotated[State1, Field(description="State of CVE - PUBLISHED, REJECTED.")]
     dateReserved: Annotated[
-        Optional[Timestamp],
+        Optional[AwareDatetime],
         Field(
             None,
             description="The date/time this CVE ID was reserved in the CVE automation workgroup services system. Disclaimer: This date reflects when the CVE ID was reserved, and does not necessarily indicate when this vulnerability was discovered, shared with the affected vendor, publicly disclosed, or updated in CVE.",
@@ -617,7 +607,7 @@ class ProviderMetadata(BaseModel):
         Field(None, description="The container provider's organizational short name."),
     ]
     dateUpdated: Annotated[
-        Optional[Timestamp],
+        Optional[AwareDatetime],
         Field(
             None,
             description="Timestamp to be set by the system of record at time of submission. If dateUpdated is provided to the system of record it will be replaced by the current timestamp at the time of submission.",
@@ -923,7 +913,7 @@ class Exploits(RootModel[List[Description]]):
 
 class TimelineItem(BaseModel):
     time: Annotated[
-        Timestamp,
+        AwareDatetime,
         Field(
             description="Timestamp representing when the event in the timeline occurred. The timestamp format is based on RFC3339 and ISO ISO8601, with an optional timezone. yyyy-MM-ddTHH:mm:ssZZZZ - if the timezone offset is not given, GMT (0000) is assumed."
         ),
@@ -1222,14 +1212,14 @@ class CnaPublishedContainer(BaseModel):
     )
     providerMetadata: ProviderMetadata
     dateAssigned: Annotated[
-        Optional[Timestamp],
+        Optional[AwareDatetime],
         Field(
             None,
             description="The date/time this CVE ID was associated with a vulnerability by a CNA.",
         ),
     ]
     datePublic: Annotated[
-        Optional[Timestamp],
+        Optional[AwareDatetime],
         Field(
             None,
             description="If known, the date/time the vulnerability was disclosed publicly.",
@@ -1270,7 +1260,7 @@ class AdpContainer(BaseModel):
     )
     providerMetadata: ProviderMetadata
     datePublic: Annotated[
-        Optional[Timestamp],
+        Optional[AwareDatetime],
         Field(
             None,
             description="If known, the date/time the vulnerability was disclosed publicly.",

vdb/lib/db6.py

@@ -42,8 +42,10 @@ def get(db_file: str = config.VDB_BIN_FILE, index_file: str = config.VDB_BIN_IND
 def clear_all():
     if db_conn:
         db_conn.execute("DELETE FROM cve_data")
+        db_conn.commit()
     if index_conn:
         index_conn.execute("DELETE FROM cve_index")
+        index_conn.commit()
 
 
 def close_all():