/* EP08 - storage hierarchy on ALINX PI-AR035-F1-1 (XC7A35T-FGG484-2).
 *
 * Compares on-chip BRAM and board DDR3:
 *   - CPU read bandwidth and CPU word-copy bandwidth in a 32 KiB scratch BRAM
 *   - hardware word-copy DMA in the same scratch BRAM (CPU idle)
 *   - dependent-load latency of the scratch BRAM (pointer chase)
 *   - DDR3 native-UI DMA write-only / read-only / CRC bandwidth at 64 MiB
 *   - DDR3 single-beat read latency, sampled across banks/rows
 *
 * CPU: CV32E20 at 50 MHz. DDR3 user clock: 100 MHz, 128-bit native UI.
 * The CPU has no direct DDR3 window; DDR3 is reached only through the DMA.
 */
#include <stdint.h>

#define UART_BASE   0x10000000UL
#define UART_TX     (*(volatile unsigned char *)(UART_BASE + 0x00))
#define UART_STAT   (*(volatile uint32_t *)(UART_BASE + 0x04))

#define DDR_BASE    0x20010000UL
#define DDR_CTRL    (*(volatile uint32_t *)(DDR_BASE + 0x00))
#define DDR_STATUS  (*(volatile uint32_t *)(DDR_BASE + 0x04))
#define DDR_SIZE    (*(volatile uint32_t *)(DDR_BASE + 0x08))
#define DDR_WR_CYC  (*(volatile uint32_t *)(DDR_BASE + 0x0c))
#define DDR_RD_CYC  (*(volatile uint32_t *)(DDR_BASE + 0x10))
#define DDR_CRC_CYC (*(volatile uint32_t *)(DDR_BASE + 0x14))
#define DDR_BEATS   (*(volatile uint32_t *)(DDR_BASE + 0x18))
#define DDR_ERRORS  (*(volatile uint32_t *)(DDR_BASE + 0x1c))
#define DDR_FERR    (*(volatile uint32_t *)(DDR_BASE + 0x20))
#define DDR_CRC_D   (*(volatile uint32_t *)(DDR_BASE + 0x24))
#define DDR_CRC_E   (*(volatile uint32_t *)(DDR_BASE + 0x28))
#define DDR_VER     (*(volatile uint32_t *)(DDR_BASE + 0x2c))
#define DDR_PING_LAT (*(volatile uint32_t *)(DDR_BASE + 0x30))
#define DDR_PING_ADDR (*(volatile uint32_t *)(DDR_BASE + 0x34))

#define BRAM_BASE   0x20030000UL
#define BRAM_CTRL   (*(volatile uint32_t *)(BRAM_BASE + 0x00))
#define BRAM_SRC    (*(volatile uint32_t *)(BRAM_BASE + 0x04))
#define BRAM_DST    (*(volatile uint32_t *)(BRAM_BASE + 0x08))
#define BRAM_LEN    (*(volatile uint32_t *)(BRAM_BASE + 0x0c))
#define BRAM_STATUS (*(volatile uint32_t *)(BRAM_BASE + 0x10))
#define BRAM_CYC    (*(volatile uint32_t *)(BRAM_BASE + 0x14))
#define BRAM_VER    (*(volatile uint32_t *)(BRAM_BASE + 0x18))

#define SCRATCH     ((volatile uint32_t *)0x20020000UL)

#define ST_CALIB    0x01u
#define ST_BUSY     0x02u
#define ST_DONE     0x04u
#define ST_FAIL     0x08u
#define ST_CRC_OK   0x10u

static void uart_putc(char c)
{
    while (UART_STAT & 1u) { }
    UART_TX = (unsigned char)c;
}

static void uart_puts(const char *s)
{
    while (*s) uart_putc(*s++);
}

static void uart_putu(uint32_t v)
{
    char b[11];
    int i = 0;
    if (v == 0u) { uart_putc('0'); return; }
    while (v) { b[i++] = (char)('0' + (v % 10u)); v /= 10u; }
    while (i) uart_putc(b[--i]);
}

static void uart_putu64(uint64_t v)
{
    char b[21];
    int i = 0;
    if (v == 0u) { uart_putc('0'); return; }
    while (v) { b[i++] = (char)('0' + (int)(v % 10u)); v /= 10u; }
    while (i) uart_putc(b[--i]);
}

static void uart_puthex(uint32_t v)
{
    int i;
    uart_puts("0x");
    for (i = 7; i >= 0; i--)
        uart_putc("0123456789abcdef"[(v >> (i * 4)) & 0xfu]);
}

static inline uint32_t rd_lo(void)
{
    uint32_t v;
    __asm__ volatile ("csrr %0, mcycle" : "=r"(v));
    return v;
}

static inline uint32_t rd_hi(void)
{
    uint32_t v;
    __asm__ volatile ("csrr %0, mcycleh" : "=r"(v));
    return v;
}

static uint64_t cycles64(void)
{
    uint32_t hi, lo, hi2;
    do {
        hi  = rd_hi();
        lo  = rd_lo();
        hi2 = rd_hi();
    } while (hi != hi2);
    return ((uint64_t)hi << 32) | lo;
}

static uint32_t mbps(uint64_t bytes, uint64_t cycles)
{
    if (cycles == 0u) return 0u;
    /* CPU cycles are 20 ns: bytes / (cycles * 20 ns) / 1e6 = bytes * 50 / cycles */
    return (uint32_t)(bytes * 50u / cycles);
}

static uint32_t mbps_ui(uint64_t bytes, uint64_t cycles)
{
    if (cycles == 0u) return 0u;
    /* MIG ui_clk cycles are 10 ns: bytes * 100 / cycles */
    return (uint32_t)(bytes * 100u / cycles);
}

static void delay_cycles(uint32_t n)
{
    volatile uint32_t i;
    for (i = 0; i < n; i++) { }
}

static void wait_ddr(void)
{
    uint64_t start = cycles64();
    uint32_t status;
    delay_cycles(64);
    do {
        status = DDR_STATUS;
        if (status & ST_FAIL) break;
        if (status & ST_DONE) break;
    } while (cycles64() - start < 1000000000ull);  /* 20 s guard covers CRC */
}

/* ------------------------------------------------------------------ */
/* BRAM tests                                                          */
/* ------------------------------------------------------------------ */

static int bram_cpu_copy(uint32_t words, uint32_t src, uint32_t dst)
{
    volatile uint32_t *s = SCRATCH + src;
    volatile uint32_t *d = SCRATCH + dst;
    uint64_t t0, t1;
    uint32_t i;
    int bad = 0;

    t0 = cycles64();
    for (i = 0; i < words; i++)
        d[i] = s[i];
    t1 = cycles64();

    for (i = 0; i < words; i++)
        if (d[i] != (uint32_t)(0xA5A50000u + i)) bad++;

    uart_puts("EP08 BRAM CPU_COPY bytes=");
    uart_putu(words * 4u);
    uart_puts(" cycles=");
    uart_putu64(t1 - t0);
    uart_puts(" MBps=");
    uart_putu(mbps((uint64_t)words * 4u, t1 - t0));
    uart_puts(" bad=");
    uart_putu((uint32_t)bad);
    uart_putc('\r'); uart_putc('\n');
    return bad;
}

static int bram_dma_copy(uint32_t words, uint32_t src, uint32_t dst)
{
    uint32_t i, status, cyc, guard;
    uint64_t t0;
    int bad = 0;

    BRAM_SRC = src;
    BRAM_DST = dst;
    BRAM_LEN = words;
    delay_cycles(16);
    BRAM_CTRL = 0x1u;                 /* start toggle */

    t0 = cycles64();
    do {
        status = BRAM_STATUS;
    } while (!(status & 0x2u) && (cycles64() - t0 < 200000ull));
    if (!(status & 0x2u)) bad = 1;

    guard = 0;
    do {
        status = BRAM_STATUS;
        guard++;
    } while ((status & 0x2u) && !(status & 0x1u) && guard < 2000000u);
    if (!(status & 0x1u)) bad = 1;

    cyc = BRAM_CYC;
    for (i = 0; i < words; i++)
        if (SCRATCH[dst + i] != (uint32_t)(0xA5A50000u + i)) bad++;

    uart_puts("EP08 BRAM DMA_COPY bytes=");
    uart_putu(words * 4u);
    uart_puts(" cycles=");
    uart_putu(cyc);
    uart_puts(" MBps=");
    uart_putu(mbps((uint64_t)words * 4u, cyc));
    uart_puts(" bad=");
    uart_putu((uint32_t)bad);
    uart_putc('\r'); uart_putc('\n');
    return bad;
}

static void bram_cpu_read(uint32_t words)
{
    volatile uint32_t *s = SCRATCH;
    uint64_t t0, t1, sum = 0;
    uint32_t i;

    t0 = cycles64();
    for (i = 0; i < words; i++)
        sum += s[i];
    t1 = cycles64();

    uart_puts("EP08 BRAM CPU_READ bytes=");
    uart_putu(words * 4u);
    uart_puts(" cycles=");
    uart_putu64(t1 - t0);
    uart_puts(" MBps=");
    uart_putu(mbps((uint64_t)words * 4u, t1 - t0));
    uart_puts(" sum=");
    uart_puthex((uint32_t)sum);
    uart_putc('\r'); uart_putc('\n');
}

/* Dependent-load pointer chase: p = *(uint32_t *)p. */
static void bram_latency(uint32_t base_word, uint32_t nodes, uint32_t laps)
{
    volatile uint32_t *ring = SCRATCH + base_word;
    uint32_t i;
    uint64_t t0, t1, cyc;
    volatile uint32_t *p = ring;

    for (i = 0; i < nodes; i++)
        ring[i] = (uint32_t)(SCRATCH + base_word + ((i + 1u) % nodes));

    t0 = cycles64();
    for (i = 0; i < nodes * laps / 8u; i++) {
        p = (volatile uint32_t *)*p;
        p = (volatile uint32_t *)*p;
        p = (volatile uint32_t *)*p;
        p = (volatile uint32_t *)*p;
        p = (volatile uint32_t *)*p;
        p = (volatile uint32_t *)*p;
        p = (volatile uint32_t *)*p;
        p = (volatile uint32_t *)*p;
    }
    t1 = cycles64();
    cyc = t1 - t0;

    uart_puts("EP08 BRAM LATENCY loads=");
    uart_putu(nodes * laps);
    uart_puts(" cycles=");
    uart_putu64(cyc);
    uart_puts(" cyc_per_load_x100=");
    uart_putu((uint32_t)(cyc * 100u / (nodes * laps)));
    uart_putc('\r'); uart_putc('\n');
}

/* ------------------------------------------------------------------ */
/* DDR3 tests                                                          */
/* ------------------------------------------------------------------ */

static int ddr_run(uint32_t wr, uint32_t rd, uint32_t crc, uint32_t bytes,
                   const char *name)
{
    uint32_t status, wr_cyc, rd_cyc, crc_cyc, beats, errs;
    int bad = 0;

    DDR_CTRL = 0x1u | (3u << 1) | (crc << 4) | (wr << 5) | (rd << 6);
    wait_ddr();

    status  = DDR_STATUS;
    wr_cyc  = DDR_WR_CYC;
    rd_cyc  = DDR_RD_CYC;
    crc_cyc = DDR_CRC_CYC;
    beats   = DDR_BEATS;
    errs    = DDR_ERRORS;

    uart_puts("EP08 DDR ");
    uart_puts(name);
    uart_puts(" bytes=");
    uart_putu(bytes);
    uart_puts(" status=");
    uart_puthex(status);
    if (wr) {
        uart_puts(" wr_cycles=");
        uart_putu(wr_cyc);
        uart_puts(" wr_MBps=");
        uart_putu(mbps_ui(bytes, wr_cyc));
    }
    if (rd) {
        uart_puts(" rd_cycles=");
        uart_putu(rd_cyc);
        uart_puts(" rd_MBps=");
        uart_putu(mbps_ui(bytes, rd_cyc));
    }
    if (crc) {
        uart_puts(" crc_cycles=");
        uart_putu(crc_cyc);
        uart_puts(" crc_MBps=");
        uart_putu(mbps_ui(bytes, crc_cyc));
        uart_puts(" crc_data=");
        uart_puthex(DDR_CRC_D);
        uart_puts(" crc_exp=");
        uart_puthex(DDR_CRC_E);
    }
    uart_puts(" beats=");
    uart_putu(beats);
    uart_puts(" err=");
    uart_putu(errs);
    uart_putc('\r'); uart_putc('\n');

    if (!(status & ST_DONE)) bad = 1;
    if (status & ST_FAIL) bad = 1;
    if (errs != 0u) bad = 1;
    if (rd && beats != (bytes >> 4)) bad = 1;
    if (crc && (DDR_CRC_D != DDR_CRC_E)) bad = 1;
    return bad;
}

static void ddr_ping(void)
{
    uint32_t i, lat, addr, min = 0xFFFFFFFFu, max = 0, sum = 0;
    uint32_t last = 0;

    for (i = 0; i < 16; i++) {
        DDR_CTRL = 0x1u | (3u << 1) | (1u << 7);   /* ping toggle */
        wait_ddr();
        lat  = DDR_PING_LAT;
        addr = DDR_PING_ADDR;
        last = addr;
        if (lat < min) min = lat;
        if (lat > max) max = lat;
        sum += lat;
    }

    uart_puts("EP08 DDR PING n=16 min_ui_cycles=");
    uart_putu(min);
    uart_puts(" max_ui_cycles=");
    uart_putu(max);
    uart_puts(" avg_x100=");
    uart_putu(sum * 100u / 16u);
    uart_puts(" avg_ns=");
    uart_putu(sum * 10u / 16u);
    uart_puts(" last_addr=");
    uart_puthex(last);
    uart_putc('\r'); uart_putc('\n');
}

int main(void)
{
    uint32_t i;
    int fail = 0;
    int round;

    uart_puts("\r\nEP08 storage hierarchy on ALINX PI-AR035-F1-1\r\n");
    uart_puts("EP08 VERSION ddr=");
    uart_puthex(DDR_VER);
    uart_puts(" bram=");
    uart_puthex(BRAM_VER);
    uart_puts(" DDR_SIZE=");
    uart_putu(DDR_SIZE);
    uart_puts(" CPU_HZ=50000000 UI_HZ=100000000\r\n");

    /* Fill the source region with an address-tagged pattern. */
    for (i = 0; i < 2048u; i++)
        SCRATCH[i] = 0xA5A50000u + i;
    for (i = 0; i < 2048u; i++)
        SCRATCH[2048u + i] = 0u;

    bram_cpu_read(8192u);
    bram_latency(4096u, 1024u, 32u);

    if (bram_cpu_copy(2048u, 0u, 2048u) != 0) fail = 1;
    if (bram_dma_copy(2048u, 0u, 2048u) != 0) fail = 1;

    /* DDR3 must be calibrated before any transaction. */
    {
        uint64_t start = cycles64();
        while ((DDR_STATUS & ST_CALIB) == 0u) {
            if (cycles64() - start > 250000000ull) {
                uart_puts("EP08 CALIB: FAIL\r\n");
                uart_puts("EP08 RESULT: FAIL\r\n");
                for (;;) { }
            }
        }
    }
    uart_puts("EP08 CALIB: PASS\r\n");

    for (round = 1; round <= 2; round++) {
        uart_puts("EP08 ROUND ");
        uart_putu((uint32_t)round);
        uart_puts(" BEGIN\r\n");

        if (ddr_run(1u, 0u, 0u, 64u * 1024u * 1024u, "WRITE") != 0) fail = 1;
        if (ddr_run(0u, 1u, 0u, 64u * 1024u * 1024u, "READ") != 0) fail = 1;
        if (ddr_run(0u, 1u, 1u, 64u * 1024u * 1024u, "CRC") != 0) fail = 1;
        if (round == 1)
            ddr_ping();

        uart_puts("EP08 ROUND ");
        uart_putu((uint32_t)round);
        uart_puts(fail ? " FAIL\r\n" : " PASS\r\n");
    }

    uart_puts(fail ? "EP08 RESULT: FAIL\r\n" : "EP08 RESULT: PASS\r\n");
    for (;;) { }
    return 0;
}
