enable SMP on GR712RC

hal/ia32/Makefile

@@ -5,7 +5,9 @@
 # Author: Pawel Pisarczyk
 #
 
-OBJS += $(addprefix $(PREFIX_O)hal/ia32/, _init.o _exceptions.o _interrupts.o multiboot.o spinlock.o exceptions.o interrupts.o cpu.o pmap.o timer.o hal.o string.o pci.o tlb.o init.o)
+include hal/tlb/Makefile
+
+OBJS += $(addprefix $(PREFIX_O)hal/ia32/, _init.o _exceptions.o _interrupts.o multiboot.o spinlock.o exceptions.o interrupts.o cpu.o pmap.o timer.o hal.o string.o pci.o init.o)
 CFLAGS += -Ihal/ia32
 
 ifeq ($(CONSOLE), vga)

hal/ia32/tlb.h → hal/ia32/arch/tlb.h

@@ -6,22 +6,27 @@
  * TLB handling
  *
  * Copyright 2023 Phoenix Systems
- * Author; Andrzej Stalke
+ * Author: Andrzej Stalke, Lukasz Leczkowski
  *
  * This file is part of Phoenix-RTOS.
  *
  * %LICENSE%
  */
 
-#ifndef _HAL_TLB_H_
-#define _HAL_TLB_H_
 
-#include <arch/types.h>
-#include "hal/spinlock.h"
+#ifndef _HAL_IA32_TLB_H_
+#define _HAL_IA32_TLB_H_
 
-static inline void hal_tlbFlushLocal(void)
+
+#include "pmap.h"
+#include "hal/tlb/tlb.h"
+
+
+static inline void hal_tlbFlushLocal(const pmap_t *pmap)
 {
 	u32 tmpreg;
+	(void)pmap;
+
 	/* clang-format off */
 	__asm__ volatile (
 		"movl %%cr3, %0\n\t"
@@ -35,8 +40,10 @@ static inline void hal_tlbFlushLocal(void)
 }
 
 
-static inline void hal_tlbInvalidateLocalEntry(void const *vaddr)
+static inline void hal_tlbInvalidateLocalEntry(const pmap_t *pmap, const void *vaddr)
 {
+	(void)pmap;
+
 	/* clang-format off */
 	__asm__ volatile (
 		"invlpg (%0)"
@@ -48,18 +55,5 @@ static inline void hal_tlbInvalidateLocalEntry(void const *vaddr)
 	return;
 }
 
-void hal_tlbFlush(void);
-
-
-void hal_tlbInvalidateEntry(const void *vaddr, size_t count);
-
-
-void hal_tlbCommit(spinlock_t *spinlock, spinlock_ctx_t *ctx);
-
-
-void hal_tlbShootdown(void);
-
-
-void hal_tlbInitCore(const unsigned int id);
 
 #endif

hal/ia32/cpu.c

@@ -21,12 +21,15 @@
 #include "hal/string.h"
 #include "hal/pmap.h"
 #include "hal/hal.h"
+#include "hal/tlb/tlb.h"
 #include "pci.h"
 #include "ia32.h"
 #include "halsyspage.h"
-#include "tlb.h"
 #include "init.h"
 
+#include <arch/tlb.h>
+
+
 struct cpu_feature_t {
 	const char *name;
 	s32 eax;
@@ -201,7 +204,7 @@ void hal_cpuSigreturn(void *kstack, void *ustack, cpu_context_t **ctx)
 
 void hal_longjmp(cpu_context_t *ctx)
 {
-	hal_tlbFlushLocal();
+	hal_tlbFlushLocal(NULL);
 	/* clang-format off */
 	__asm__ volatile (
 		"cli\n\t"
@@ -251,7 +254,7 @@ void hal_longjmp(cpu_context_t *ctx)
 
 void hal_jmp(void *f, void *kstack, void *stack, int argc)
 {
-	hal_tlbFlushLocal();
+	hal_tlbFlushLocal(NULL);
 	if (stack == NULL) {
 		/* clang-format off */
 		__asm__ volatile (
@@ -627,7 +630,7 @@ void _hal_cpuInit(void)
 void hal_cpuTlsSet(hal_tls_t *tls, cpu_context_t *ctx)
 {
 	(void)ctx;
-	hal_tlbFlushLocal();
+	hal_tlbFlushLocal(NULL);
 	_cpu_gdtInsert(hal_cpuGetTlsIndex(), tls->tls_base + tls->tbss_sz + tls->tdata_sz, VADDR_KERNEL - tls->tls_base + tls->tbss_sz + tls->tdata_sz, DESCR_TLS);
 	/* Reload the hidden gs register*/
 	hal_cpuReloadTlsSegment();

hal/ia32/init.c

@@ -18,7 +18,8 @@
 #include "init.h"
 #include "ia32.h"
 #include "include/errno.h"
-#include "tlb.h"
+
+#include <arch/tlb.h>
 
 extern unsigned int _end;
 extern syspage_t *syspage;
@@ -79,7 +80,7 @@ static inline int _hal_configMapPage(u32 *pdir, addr_t pa, void *va, int attr)
 			if (ret == 0) {
 				ptable = (addr_t *)(syspage->hs.ptable + VADDR_KERNEL);
 				ptable[((u32)hal_config.ptable >> 12) & 0x000003ffu] = (page.addr & ~(SIZE_PAGE - 1)) | (PGHD_WRITE | PGHD_PRESENT);
-				hal_tlbInvalidateLocalEntry(hal_config.ptable);
+				hal_tlbInvalidateLocalEntry(NULL, hal_config.ptable);
 				hal_memset(hal_config.ptable, 0, SIZE_PAGE);
 				ret = _pmap_enter(pdir, hal_config.ptable, pa, va, attr, &page, 0);
 			}

hal/ia32/interrupts.c

@@ -23,9 +23,10 @@
 
 #include "proc/userintr.h"
 #include "include/errno.h"
-#include "tlb.h"
 #include "init.h"
 
+#include <arch/tlb.h>
+
 
 /* Hardware interrupt stubs */
 extern void _interrupts_irq0(void);

hal/ia32/pmap.c

@@ -14,13 +14,15 @@
  * %LICENSE%
  */
 
+#include <arch/tlb.h>
+
 #include "halsyspage.h"
 #include "ia32.h"
 #include "hal/pmap.h"
 #include "hal/spinlock.h"
 #include "hal/string.h"
 #include "hal/console.h"
-#include "tlb.h"
+#include "hal/tlb/tlb.h"
 #include "init.h"
 
 #include "include/errno.h"
@@ -103,16 +105,16 @@ int _pmap_enter(u32 *pdir, addr_t *pt, addr_t pa, void *va, int attr, page_t *al
 	ptable = (addr_t *)(syspage->hs.ptable + VADDR_KERNEL);
 	ptable[((u32)pt >> 12) & 0x000003ffu] = (addr & ~(SIZE_PAGE - 1)) | (PGHD_WRITE | PGHD_PRESENT | PGHD_USER);
 
-	hal_tlbInvalidateLocalEntry(pt);
+	hal_tlbInvalidateLocalEntry(NULL, pt);
 
 	/* And at last map page or only changle attributes of map entry */
 	pt[pti] = ((pa & ~(SIZE_PAGE - 1)) | (attr & 0xfffu) | PGHD_PRESENT);
 
 	if (tlbInval != 0) {
-		hal_tlbInvalidateEntry(va, 1);
+		hal_tlbInvalidateEntry(NULL, va, 1);
 	}
 	else {
-		hal_tlbInvalidateLocalEntry(va);
+		hal_tlbInvalidateLocalEntry(NULL, va);
 	}
 
 	return EOK;
@@ -165,18 +167,18 @@ int _pmap_removeMany(u32 *pdir, addr_t *pt, void *vaddr, size_t count, int tlbIn
 		/* Map selected page table */
 		ptable[((u32)pt >> 12) & 0x000003ffu] = (addr & ~(SIZE_PAGE - 1)) | (PGHD_WRITE | PGHD_PRESENT);
 
-		hal_tlbInvalidateLocalEntry(pt);
+		hal_tlbInvalidateLocalEntry(NULL, pt);
 
 		/* Unmap page */
 		pt[pti] = 0;
 	}
 
 	if (tlbInval != 0) {
-		hal_tlbInvalidateEntry(vaddr, count);
+		hal_tlbInvalidateEntry(NULL, vaddr, count);
 	}
 	else {
 		for (i = 0; i < count; ++i, vaddr += SIZE_PAGE) {
-			hal_tlbInvalidateLocalEntry(vaddr);
+			hal_tlbInvalidateLocalEntry(NULL, vaddr);
 		}
 	}
 	return EOK;
@@ -226,7 +228,7 @@ addr_t pmap_resolve(pmap_t *pmap, void *vaddr)
 
 	ptable = (addr_t *)(syspage->hs.ptable + VADDR_KERNEL);
 	ptable[((u32)hal_config.ptable >> 12) & 0x000003ffu] = (addr & ~(SIZE_PAGE - 1)) | (PGHD_WRITE | PGHD_PRESENT);
-	hal_tlbInvalidateLocalEntry(hal_config.ptable);
+	hal_tlbInvalidateLocalEntry(NULL, hal_config.ptable);
 
 	addr = (addr_t)hal_config.ptable[pti];
 	hal_tlbCommit(&pmap_common.lock, &sc);
@@ -350,7 +352,7 @@ int _pmap_kernelSpaceExpand(pmap_t *pmap, void **start, void *end, page_t *dp)
 		}
 		*start = vaddr;
 	}
-	hal_tlbFlushLocal();
+	hal_tlbFlushLocal(NULL);
 
 	pmap->start = (void *)VADDR_KERNEL;
 	pmap->end = end;
@@ -411,7 +413,7 @@ void _pmap_init(pmap_t *pmap, void **vstart, void **vend)
 	pmap->start = (void *)VADDR_KERNEL;
 	pmap->end = (void *)VADDR_MAX;
 
-	hal_tlbFlushLocal();
+	hal_tlbFlushLocal(NULL);
 
 	/* Initialize kernel heap start address */
 	(*vstart) = hal_config.heapStart;
@@ -424,7 +426,7 @@ void _pmap_init(pmap_t *pmap, void **vstart, void **vend)
 	(*vstart) += 0x500;
 
 	pmap_removeMany(pmap, *vend, ((void *)VADDR_KERNEL + (4 << 20) - *vend) / SIZE_PAGE);
-	hal_tlbFlushLocal();
+	hal_tlbFlushLocal(NULL);
 
 	return;
 }

hal/sparcv8leon3/Makefile

@@ -13,5 +13,5 @@ OBJS += $(addprefix $(PREFIX_O)hal/$(TARGET_SUFF)/, cpu.o hal.o spinlock.o strin
 ifeq ($(findstring -DNOMMU,$(CPPFLAGS)),-DNOMMU)
   OBJS += $(addprefix $(PREFIX_O)hal/$(TARGET_SUFF)/, exceptions-nommu.o pmap-nommu.o _interrupts-nommu.o)
 else
-  OBJS += $(addprefix $(PREFIX_O)hal/$(TARGET_SUFF)/, exceptions.o pmap.o srmmu.o _interrupts.o)
+  OBJS += $(addprefix $(PREFIX_O)hal/$(TARGET_SUFF)/, exceptions.o pmap.o srmmu.o tlb.o _interrupts.o)
 endif

hal/sparcv8leon3/_interrupts-nommu.S

@@ -136,6 +136,8 @@ irq_kstack_set:
 	nop
 	nop
 
+	MULTILOCK_CLEAR
+
 	/* l7 still points to bottom of context */
 	mov %l7, %sp
 

hal/sparcv8leon3/_interrupts.S

@@ -160,6 +160,8 @@ irq_cwp_done0:
 	ta 0xb
 #endif
 
+	MULTILOCK_CLEAR
+
 	/* l7 still points to bottom of context */
 	mov %l7, %sp
 

hal/sparcv8leon3/_traps.S

@@ -398,11 +398,11 @@ exc_wovfl_done:
 	ld [%l6 + %l7], %l7
 	ba exc_kstack_set
 	sub %l7, 0x60, %sp
- 
+
  exc_no_kstack_switch:
 	/* we came from kernel, make space for context */
 	sub %fp, 0x60, %sp
- 
+
  exc_kstack_set:
 	/* Save context on kernel stack */
 
@@ -450,7 +450,7 @@ exc_wovfl_done:
 	/* trap type */
 	set TBR_TT_MSK, %g2
 	and %g1, %g2, %o0
-	
+
 	/* void exceptions_dispatch(unsigned int n, exc_context_t *ctx) */
 	call exceptions_dispatch
 	srl %o0, TBR_TT_SHIFT, %o0
@@ -954,6 +954,8 @@ r_return:
 	ta 0xb
 #endif
 
+	MULTILOCK_CLEAR
+
 	ld [%l1], %g2 /* ctx pointer */
 
 	/* Set %psr of the new task.

hal/sparcv8leon3/arch/cpu.h

@@ -69,6 +69,21 @@
 #define CCR_FD  (1 << 22) /* Flush DCache */
 #define CCR_DS  (1 << 23) /* DCache snooping */
 
+/* Basic address space identifiers */
+#define ASI_USER_INSTR  0x08
+#define ASI_SUPER_INSTR 0x09
+#define ASI_USER_DATA   0x0a
+#define ASI_SUPER_DATA  0x0b
+
+/* Interrupts multilock */
+
+/* clang-format off */
+#define MULTILOCK_CLEAR \
+	stbar; \
+	set hal_multilock, %g1; \
+	stub %g0, [%g1]
+/* clang-format on */
+
 
 #ifndef __ASSEMBLY__
 
@@ -77,6 +92,8 @@
 #include "gaisler/gaisler.h"
 
 
+#define MAX_CPU_COUNT NUM_CPUS
+
 #define SYSTICK_INTERVAL 1000
 
 
@@ -286,6 +303,29 @@ static inline void hal_cpuEnableInterrupts(void)
 }
 
 
+static inline void hal_cpuAtomicInc(volatile u32 *dst)
+{
+	/* clang-format off */
+
+	__asm__ volatile (
+		"ld [%0], %%g1\n\t"
+	"1: \n\t"
+		"mov %%g1, %%g2\n\t"
+		"inc %%g1\n\t"
+	".align 16\n\t" /* GRLIB TN-0011 errata */
+		"casa [%0] %c1, %%g2, %%g1\n\t"
+		"cmp %%g1, %%g2\n\t"
+		"bne 1b\n\t"
+		"nop\n\t"
+		:
+		: "r"(dst), "i"(ASI_SUPER_DATA)
+		: "g1", "g2", "memory"
+	);
+
+	/* clang-format on */
+}
+
+
 #endif /* __ASSEMBLY__ */
 
 

hal/sparcv8leon3/arch/tlb.h

@@ -0,0 +1,34 @@
+/*
+ * Phoenix-RTOS
+ *
+ * Operating system kernel
+ *
+ * TLB handling
+ *
+ * Copyright 2023 Phoenix Systems
+ * Author: Andrzej Stalke, Lukasz Leczkowski
+ *
+ * This file is part of Phoenix-RTOS.
+ *
+ * %LICENSE%
+ */
+
+
+#ifndef _HAL_LEON3_TLB_H_
+#define _HAL_LEON3_TLB_H_
+
+#include "cpu.h"
+#include "pmap.h"
+#include "hal/tlb/tlb.h"
+
+
+void hal_tlbFlushLocal(const pmap_t *pmap);
+
+
+void hal_tlbInvalidateLocalEntry(const pmap_t *pmap, const void *vaddr);
+
+
+int hal_tlbIrqHandler(unsigned int n, cpu_context_t *ctx, void *arg);
+
+
+#endif