first progress on frontend

build.zig

@@ -2,7 +2,7 @@ const std = @import("std");
 
 const cases = @import("tests/cases.zig");
 
-var trace: ?bool = false;
+var trace: bool = false;
 var @"enable-bench": ?bool = false;
 var backend: TraceBackend = .None;
 
@@ -19,9 +19,9 @@ pub fn build(b: *std.Build) !void {
 
     trace = b.option(bool, "trace",
         \\Enables tracing of the compiler using the default backend (spall)
-    );
+    ) orelse false;
 
-    if (trace) |_| {
+    if (trace) {
         backend = b.option(TraceBackend, "trace-backend",
             \\Switch between what backend to use. None is default.
         ) orelse backend;
@@ -40,7 +40,7 @@ pub fn build(b: *std.Build) !void {
 
     const exe_options = b.addOptions();
 
-    exe_options.addOption(bool, "trace", trace orelse false);
+    exe_options.addOption(bool, "trace", trace);
     exe_options.addOption(TraceBackend, "backend", backend);
     exe_options.addOption(std.log.Level, "debug_log", debug_log);
     exe_options.addOption(usize, "src_file_trimlen", std.fs.path.dirname(std.fs.path.dirname(@src().file).?).?.len);

demo/print_ast.py

@@ -1,7 +0,0 @@
-
-import marshal
-
-filename = './demo/test.py'
-with open(filename, 'r') as f:
-    bytes = marshal.load(f)
-

demo/test.py

@@ -1,5 +1 @@
-# Set
-a = {1, 2, 3, 4, 5}
-# a.add(6)
-
-print(a)
+a = 1

src/Manager.zig

@@ -7,8 +7,7 @@ const Manager = @This();
 
 const tracer = @import("tracer");
 
-// const Tokenizer = @import("frontend/tokenizer/Tokenizer.zig");
-// const Parser = @import("frontend/Parser.zig");
+const Ast = @import("frontend/Ast.zig");
 
 const Marshal = @import("compiler/Marshal.zig");
 const Vm = @import("vm/Vm.zig");
@@ -31,7 +30,6 @@ pub fn run_pyc(manager: *Manager, file_name: []const u8) !void {
 
     // Open source file.
     const source_file = try std.fs.cwd().openFile(file_name, .{});
-
     const source_file_size = (try source_file.stat()).size;
 
     const source = try source_file.readToEndAllocOptions(
@@ -50,34 +48,19 @@ pub fn run_pyc(manager: *Manager, file_name: []const u8) !void {
 }
 
 pub fn run_file(manager: *Manager, file_name: []const u8) !void {
-    _ = std.ChildProcess.run(.{
-        .allocator = manager.allocator,
-        .argv = &.{
-            "python3.10",
-            "-m",
-            "py_compile",
-            file_name,
-        },
-        .cwd = ".",
-        .expand_arg0 = .expand,
-    }) catch @panic("failed to side-run python");
-
-    // This outputs to __pycache__/file_name.cpython-310.pyc
-    const output_file_name: []const u8 = name: {
-        const trimmed_name: []const u8 = file_name[0 .. file_name.len - ".py".len];
-        const output_file = std.fs.path.basename(trimmed_name);
-
-        log.debug("Trimmed: {s}", .{trimmed_name});
-
-        const output_dir = std.fs.path.dirname(trimmed_name) orelse @panic("why in root");
-
-        const output_pyc = try std.fmt.allocPrint(manager.allocator, "{s}/__pycache__/{s}.cpython-310.pyc", .{ output_dir, output_file });
-
-        break :name output_pyc;
-    };
+    const source_file = try std.fs.cwd().openFile(file_name, .{ .lock = .exclusive });
+    defer source_file.close();
+
+    const source_file_size = (try source_file.stat()).size;
 
-    log.debug("File: {s}", .{output_file_name});
+    const source = try source_file.readToEndAllocOptions(
+        manager.allocator,
+        source_file_size,
+        source_file_size,
+        @alignOf(u8),
+        0,
+    );
 
-    // Run python on that.
-    try manager.run_pyc(output_file_name);
+    const ast = try Ast.parse(source, manager.allocator);
+    _ = ast;
 }

src/frontend/Ast.zig

@@ -0,0 +1,58 @@
+//! Generates an AST given python source code.
+
+source: [:0]const u8,
+tokens: TokenList.Slice,
+nodes: NodeList.Slice,
+
+pub const NodeList = std.MultiArrayList(Parser.Node);
+pub const TokenList = std.MultiArrayList(Token);
+
+pub const TokenIndex = u32;
+
+pub fn parse(source: [:0]const u8, allocator: Allocator) !Ast {
+    var tokens: std.MultiArrayList(Token) = .{};
+    defer tokens.deinit(allocator);
+
+    var tokenizer = Tokenizer.init(source);
+    while (true) {
+        const token = tokenizer.next();
+        log.debug("Token: {}", .{token.tag});
+        try tokens.append(allocator, .{
+            .tag = token.tag,
+            .start = @as(u32, @intCast(token.loc.start)),
+        });
+        if (token.tag == .eof) break;
+    }
+
+    var parser = Parser{
+        .tokens = tokens,
+        .token_index = 0,
+        .allocator = allocator,
+        .nodes = .{},
+        .source = source,
+    };
+    defer parser.tokens.deinit(allocator);
+    defer parser.nodes.deinit(allocator);
+
+    try parser.parseFile();
+
+    return Ast{
+        .source = source,
+        .tokens = tokens.toOwnedSlice(),
+        .nodes = parser.nodes.toOwnedSlice(),
+    };
+}
+
+pub const Token = struct {
+    tag: Tokenizer.Token.Tag,
+    start: u32,
+};
+
+const Parser = @import("Parser.zig");
+const Tokenizer = @import("Tokenizer.zig");
+
+const log = std.log.scoped(.ast);
+
+const Ast = @This();
+const std = @import("std");
+const Allocator = std.mem.Allocator;

src/frontend/Parser.zig

@@ -0,0 +1,109 @@
+//! Converts a list of Tokens into an AST
+
+tokens: Ast.TokenList,
+nodes: Ast.NodeList,
+allocator: Allocator,
+token_index: u32 = 0,
+
+source: [:0]const u8,
+
+/// file: [statements] ENDMARKER
+pub fn parseFile(p: *Parser) !void {
+    if (p.tokens.get(p.token_index).tag == .eof) return;
+    try p.parseStatements();
+    _ = p.eatToken(.eof) orelse return error.NotEof;
+}
+
+/// statements: statement+
+fn parseStatements(p: *Parser) !void {
+    while (p.tokens.get(p.token_index).tag != .eof) {
+        try p.parseStatement();
+    }
+}
+
+/// statement: compound_stmt  | simple_stmts
+fn parseStatement(p: *Parser) !void {
+    // TODO: compound_stmt
+    try p.parseSimpleStatment();
+}
+
+fn parseSimpleStatment(p: *Parser) !void {
+    const tag = p.tokens.get(p.token_index).tag;
+    switch (tag) {
+        .identifier => {
+            const next_tag = p.tokens.get(p.token_index + 1).tag;
+            if (next_tag == .eof) {
+                @panic("simple statment found eof after ident");
+            }
+            switch (next_tag) {
+                .assign => try p.parseAssignExpr(),
+                else => std.debug.panic("TODO: parseSimpleStatment identifier {}", .{next_tag}),
+            }
+        },
+        else => std.debug.panic("TODO: parseSimpleStatment {}", .{tag}),
+    }
+}
+
+/// assignment:
+///    | NAME ':' expression ['=' annotated_rhs ]
+///    | ('(' single_target ')'
+///         | single_subscript_attribute_target) ':' expression ['=' annotated_rhs ]
+///    | (star_targets '=' )+ (yield_expr | star_expressions) !'=' [TYPE_COMMENT]
+///    | single_target augassign ~ (yield_expr | star_expressions)
+fn parseAssignExpr(p: *Parser) !void {
+    const maybe_ident_tok = p.eatToken(.identifier);
+    if (maybe_ident_tok) |ident_tok| {
+        _ = ident_tok;
+        return;
+    }
+
+    @panic("TODO: parseAssignExpr non-ident");
+}
+
+fn eatToken(p: *Parser, tag: Tokenizer.Token.Tag) ?Token {
+    const next_tok = p.nextToken();
+    if (next_tok.tag == tag) return next_tok;
+    return null;
+}
+
+fn nextToken(p: *Parser) Token {
+    const tok = p.tokens.get(p.token_index);
+    p.token_index += 1;
+    return tok;
+}
+
+fn addNode(p: *Parser, elem: Node) Allocator.Error!Node.Index {
+    const result = @as(Node.Index, @intCast(p.nodes.len));
+    try p.nodes.append(p.gpa, elem);
+    return result;
+}
+
+pub const Node = struct {
+    tag: Tag,
+    main_token: Ast.TokenIndex,
+    data: Data,
+
+    pub const Index = u32;
+
+    pub const Tag = enum(u8) {
+        root,
+        /// An assignment.
+        ///
+        /// `lhs = rhs`. main_token is the `=`.
+        assign,
+    };
+
+    pub const Data = struct {
+        lhs: Index,
+        rhs: Index,
+    };
+};
+
+const Parser = @This();
+
+const std = @import("std");
+const Ast = @import("Ast.zig");
+const Tokenizer = @import("Tokenizer.zig");
+const Token = Ast.Token;
+
+const Allocator = std.mem.Allocator;