PrincetonUniversity · alekrop · Sep 21, 2023 · Sep 21, 2023 · Sep 21, 2023 · Sep 21, 2023
diff --git a/.gitmodules b/.gitmodules
@@ -6,4 +6,7 @@
 	url = https://github.com/PrincetonUniversity/openpiton-aws.git
 [submodule "piton/design/chipset/rv64_platform/bootrom/u-boot/uboot"]
 	path = piton/design/chipset/rv64_platform/bootrom/u-boot/uboot
-	url = [email protected]:u-boot/u-boot.git
+	url = https://github.com/u-boot/u-boot.git
+[submodule "piton/design/chipset/io_ctrl/xilinx/common/ip_cores/eth_cmac_syst"]
+	path = piton/design/chipset/io_ctrl/xilinx/common/ip_cores/eth_cmac_syst
+	url = https://github.com/bsc-loca/100gb-ethernet
diff --git a/README.md b/README.md
@@ -505,3 +505,64 @@ The command will tell print the afi and agfi of your image. You can track the sy
 
 8. After the synthesis is done - you can go load it in your F1 instance!
 
+### Synthesizing OpenPiton for ALVEO boards
+
+This section has been added under MEEP project. For an eventual PR to OpenPiton team, we will need to review it.
+
+The flow is very simillar to synthesizing image for any other FPGA OpenPiton supports:
+
+1. Clone OpenPiton repo (MEEP version): git clone https://gitlab.bsc.es/hwdesign/frameworks/meep_openpiton.git
+
+2. cd into repo, run these bash commands:
+
+```
+    source piton/piton_settings.bash
+    source piton/ariane_setup.sh
+    (Follow instructions in the top of the README in case this is the first time you install OpenPiton in your machine.)
+```
+
+4. Run the synthesis:
+
+```
+    protosyn --board alveou280 --design system --core ariane --x_tiles 1 --y_tiles 1 --uart-dmw ddr --zeroer_off
+
+    Extra avaialble protosyn options:
+             --eth                     # adding CMAC based Ethernet unit
+             --ethport <num>           # optional board-level Ethernet port (default=0)
+
+             --hbm                     # define HBM as primary system memory
+             --multimc <num>           # implement design with multiple <num> connections to system memory (valid only for HBM)
+             --multimc_indices <coma separated list> # optional list of particular edge tiles for above `multimc` option
+
+             --bram-test hello_world.c # compiling and runniing VCS-based simulation before synthesis
+             --verdi-dbg  # creating Verdi compliant simulation database for above test (verdi run inside ./build dir (-sx is optional): verdi -ssf ./novas.fsdb)
+```
+
+This will create a Vivado design under $ROOT_DIR/build/...
+
+5. After the synthesis is complete (takes about 2-3 hours on fast PC), you can program the FPGA via JTAG
+
+6. Probably you would need to reboot to be able to use the new QDMA PCIe interface.
+
+7. Now you can download from the intranet the bbl containing the Linux kernel, a script to load it to the SDRAM and the bitstream itself in case you want to skip steps above.
+
+```
+    /home/fpga-runnerMEEP/lagarto_sdk_deploy/rv64gc/
+```
+
+In a separated bash window, open a client for the UART:
+
+```
+    picocom -b 115200 /dev/ttyUSB2
+```
+
+8. Clone FPGA tools repo: https://gitlab.bsc.es/hwdesign/fpga/integration-lab/fpga-tools.git
+
+Issue the next commands inside the downloaded repo:
+
+```
+    ./fpga/load-bitstream-onic.sh qdma <fpga_bistream_name>.bit
+	./boot_riscv/boot_acme.sh <osbi_buildroot>.bin
+```
+
+You should be able to see Linux booting on the other terminal.
diff --git a/piton/design/chip/rtl/chip.v.pyv b/piton/design/chip/rtl/chip.v.pyv
@@ -1,3 +1,4 @@
+// Modified by Barcelona Supercomputing Center on March 3rd, 2022
 // Modified by Princeton University on June 9th, 2015
 // ========== Copyright Header Begin ==========================================
 //
@@ -139,6 +140,17 @@ module chip(
    output                                       offchip_processor_noc3_yummy
 `endif // endif PITON_NO_CHIP_BRIDGE
 
+  `ifdef PITON_EXTRA_MEMS
+    ,
+    output [`PITON_EXTRA_MEMS * `NOC_DATA_WIDTH -1:0] processor_mcx_noc2_data,
+    output [`PITON_EXTRA_MEMS-1:0]                    processor_mcx_noc2_valid,
+    input  [`PITON_EXTRA_MEMS-1:0]                    processor_mcx_noc2_yummy,
+
+    input  [`PITON_EXTRA_MEMS * `NOC_DATA_WIDTH -1:0] mcx_processor_noc3_data,
+    input  [`PITON_EXTRA_MEMS-1:0]                    mcx_processor_noc3_valid,
+    output [`PITON_EXTRA_MEMS-1:0]                    mcx_processor_noc3_yummy
+  `endif
+
 `ifdef PITON_RV64_PLATFORM
 `ifdef PITON_RV64_DEBUGUNIT
     // Debug
@@ -1037,6 +1049,15 @@ module chip(
     assign default_total_num_tiles = `PITON_NUM_TILES;
     // Generate tile instances
 <%
+    sub_assign = r'''
+            assign default_coreid_x [_FLAT_ID_] =COREIDX;
+            assign default_coreid_y [_FLAT_ID_] =COREIDY;
+            assign flat_tileid  [_FLAT_ID_] =   _FLAT_ID_; 
+    '''
+    assign = r'''
+
+
+    '''
     template = r'''
     tile #(.TILE_TYPE(TYPE_OF_TILE))
     tile0 (
@@ -1114,6 +1135,32 @@ module chip(
     template = template[:-1] + r'''
     );
     '''
+
+
+    edge_sub1=r'''    
+    assign default_coreid_x [ENDP] = COREIDX;
+    assign default_coreid_y [ENDP] = COREIDY;
+    assign flat_tileid [ENDP]= ENDP;
+
+    assign dataOut_NN [ENDP] = endp_DatOut_NN;
+    assign validOut_NN[ENDP] = endp_ValidOut_NN;    
+    assign endp_YummyIn_NN = yummyIn_NN[ENDP];
+    '''
+    edge_sub2=r'''  
+    assign endp_DatIn_NN = dataIn_NN [ENDP];
+    assign endp_ValidIn_NN = validIn_NN[ENDP];
+    assign yummyOut_NN[ENDP] = endp_YummyOut_NN;
+
+    '''
+    edge_assign=r'''
+
+    '''   
+
+
+    MC_POS=get_mc_mapping(PITON_X_TILES,PITON_Y_TILES,PITON_MC_INDICES,PITON_EXTRA_MEMS,PITON_NETWORK_CONFIG)
+    edge_idx=len(MC_POS)
+
+
 
     # generate the tiles
     for i in range(PITON_X_TILES):
@@ -1122,10 +1169,67 @@ module chip(
             flatid = i + (j * PITON_X_TILES);
             # print template
             currenttile = template.replace("tile0", "tile%d" % (flatid));
+            # special case for core 0
+            if i == 0 and j == 0:
+               for k in [1,2,3]:
+                  tmp = edge_sub1 + edge_sub2;
+                  tmp = tmp.replace("endp_DatOut_NN","offchip_out_E_noc%d_data" %k);
+                  tmp = tmp.replace("endp_ValidOut_NN","offchip_out_E_noc%d_valid" %k);
+                  tmp = tmp.replace("endp_YummyOut_NN","offchip_out_E_noc%d_yummy" %k);
+                  tmp = tmp.replace("endp_DatIn_NN","tile_0_0_out_W_noc%d_data" %k);
+                  tmp = tmp.replace("endp_ValidIn_NN","tile_0_0_out_W_noc%d_valid" %k);
+                  tmp = tmp.replace("endp_YummyIn_NN","tile_0_0_out_W_noc%d_yummy" %k);                  
+                  tmp = tmp.replace("_NN", str(k))
+                  tmp = tmp.replace("ENDP","CHIP_SET_ID");
+                  tmp = tmp.replace("COREIDX", "8'd" + repr(i));
+                  tmp = tmp.replace("COREIDY", "8'd" + repr(j));
+
+                  edge_assign=edge_assign + tmp; 
+            # place the memory controllers           
+            for k in range(edge_idx):
+                ax = MC_POS[k]['x']
+                ay = MC_POS[k]['y']  
+                ap = MC_POS[k]['p']
+                id = MC_POS[k]['endp']                 
+                if(i==ax and j==ay):                    
+                    currenttile = currenttile.replace("out_%s_noc2_data"  % MC_POS[k]['p'], "processor_mcx_noc2_data [%d * `NOC_DATA_WIDTH +: `NOC_DATA_WIDTH]" % k);
+                    currenttile = currenttile.replace("out_%s_noc2_valid" % MC_POS[k]['p'], "processor_mcx_noc2_valid[%d]" % k);
+                    currenttile = currenttile.replace("in_%s_noc2_yummy"  % MC_POS[k]['p'], "processor_mcx_noc2_yummy[%d]" % k);
+                    currenttile = currenttile.replace("in_%s_noc3_data"   % MC_POS[k]['p'], "mcx_processor_noc3_data [%d * `NOC_DATA_WIDTH +: `NOC_DATA_WIDTH]" % k);
+                    currenttile = currenttile.replace("in_%s_noc3_valid"  % MC_POS[k]['p'], "mcx_processor_noc3_valid[%d]" % k);
+                    currenttile = currenttile.replace("out_%s_noc3_yummy" % MC_POS[k]['p'], "mcx_processor_noc3_yummy[%d]" % k);
+                    tmp = ("//Connect MC %d to enp %d:\n" % (k,id)) 
+                    tmp = tmp + edge_sub1;                    
+                    tmp = tmp.replace("ENDP",str(id));
+                    tmp = tmp.replace("COREIDX", "8'd" + repr(i));
+                    tmp = tmp.replace("COREIDY", "8'd" + repr(j));
+                    tmp = tmp.replace("endp_DatOut_NN","mcx_processor_noc3_data [%d * `NOC_DATA_WIDTH +: `NOC_DATA_WIDTH]" % k);
+                    tmp = tmp.replace("endp_ValidOut_NN","mcx_processor_noc3_valid[%d]" % k);
+                    tmp = tmp.replace("endp_YummyIn_NN","mcx_processor_noc3_yummy[%d]" % k);
+                    tmp = tmp.replace("_NN", "3");
+                    edge_assign=edge_assign + tmp;
+                    tmp = edge_sub2 ;   
+                    tmp = tmp.replace("ENDP",str(id));
+                    tmp = tmp.replace("COREIDX", "8'd" + repr(i));
+                    tmp = tmp.replace("COREIDY", "8'd" + repr(j)); 
+                    tmp = tmp.replace("endp_DatIn_NN","processor_mcx_noc2_data [%d * `NOC_DATA_WIDTH +: `NOC_DATA_WIDTH]" % k);
+                    tmp = tmp.replace("endp_ValidIn_NN","processor_mcx_noc2_valid[%d]" % k); 
+                    tmp = tmp.replace("endp_YummyOut_NN", "processor_mcx_noc2_yummy[%d]" % k); 
+                    tmp = tmp.replace("_NN", "2");
+                    edge_assign=edge_assign + tmp;
+
             currenttile = currenttile.replace("COREIDX", "8'd" + repr(i));
             currenttile = currenttile.replace("COREIDY", "8'd" + repr(j));
             currenttile = currenttile.replace("out_", "tile_%d_%d_out_" % (j,i));
             currenttile = currenttile.replace("_FLAT_ID_", repr(flatid));
+
+            assign_tmp = sub_assign
+            assign_tmp = assign_tmp.replace("COREIDX", "8'd" + repr(i));
+            assign_tmp = assign_tmp.replace("COREIDY", "8'd" + repr(j));
+            assign_tmp = assign_tmp.replace("_FLAT_ID_", repr(flatid));
+
+            assign = assign + assign_tmp;
+
             currenttype = "`SPARC_TILE"
 
             if (PITON_PICO_HET):

diff --git a/piton/design/chip/tile/ariane b/piton/design/chip/tile/ariane
diff --git a/piton/design/chip/tile/ariane_patch/ariane.patch b/piton/design/chip/tile/ariane_patch/ariane.patch
@@ -0,0 +1,36 @@
+diff --git a/Flist.ariane b/Flist.ariane
+--- a/Flist.ariane
++++ b/Flist.ariane
+@@ -76,6 +76,8 @@ src/axi_mem_if/src/axi2mem.sv
+ src/tech_cells_generic/src/pulp_clock_gating.sv
+ src/tech_cells_generic/src/cluster_clock_inverter.sv
+ src/tech_cells_generic/src/pulp_clock_mux2.sv
++src/pmp/src/pmp.sv
++src/pmp/src/pmp_entry.sv
+ src/axi_adapter.sv
+ src/alu.sv
+ src/fpu_wrap.sv
+@@ -133,8 +135,6 @@ src/riscv-dbg/src/dmi_jtag_tap.sv
+ src/riscv-dbg/debug_rom/debug_rom.sv
+ openpiton/ariane_verilog_wrap.sv
+ openpiton/riscv_peripherals.sv
+-openpiton/bootrom/baremetal/bootrom.sv
+-openpiton/bootrom/linux/bootrom_linux.sv
+ src/rv_plic/rtl/rv_plic_target.sv
+ src/rv_plic/rtl/rv_plic_gateway.sv
+ src/rv_plic/rtl/plic_regmap.sv
+diff --git a/src/frontend/frontend.sv b/src/frontend/frontend.sv
+--- a/src/frontend/frontend.sv
++++ b/src/frontend/frontend.sv
+@@ -343,10 +343,8 @@ module frontend #(
+           icache_data_q        <= icache_data;
+           icache_vaddr_q       <= icache_dreq_i.vaddr;
+           // Map the only three exceptions which can occur in the frontend to a two bit enum
+-          if (icache_dreq_i.ex.cause == riscv::INSTR_PAGE_FAULT) begin
++          if (icache_dreq_i.ex.valid) begin
+             icache_ex_valid_q <= ariane_pkg::FE_INSTR_PAGE_FAULT;
+-          end else if (icache_dreq_i.ex.cause == riscv::INSTR_ACCESS_FAULT) begin
+-            icache_ex_valid_q <= ariane_pkg::FE_INSTR_ACCESS_FAULT;
+           end else icache_ex_valid_q <= ariane_pkg::FE_NONE;
+           // save the uppermost prediction
+           btb_q                <= btb_prediction[INSTR_PER_FETCH-1];
diff --git a/piton/design/chip/tile/ariane_patch/ariane_patch.sh b/piton/design/chip/tile/ariane_patch/ariane_patch.sh
@@ -0,0 +1,4 @@
+
+echo "Patching Ariane submodule..."
+cd $ARIANE_ROOT
+git apply $DV_ROOT/design/chip/tile/ariane_patch/ariane.patch
diff --git a/piton/design/chip/tile/l15/rtl/noc1encoder.v b/piton/design/chip/tile/l15/rtl/noc1encoder.v
@@ -1,3 +1,4 @@
+// Modified by Barcelona Supercomputing Center on March 3rd, 2022
 /*
 Copyright (c) 2015 Princeton University
 All rights reserved.
@@ -436,6 +437,8 @@ begin
    msg_options_2[`MSG_CACHE_TYPE_] = msg_cache_type;
    msg_options_2[`MSG_SUBLINE_VECTOR_] = msg_subline_vector;
 
+   msg_options_3[`MSG_INI_X_] = coreid_x;
+   msg_options_3[`MSG_INI_Y_] = coreid_y;
    msg_options_3[`MSG_SDID_] = req_csm_sdid;
    msg_options_3[`MSG_LSID_] = req_csm_lsid;
 end

diff --git a/piton/design/chip/tile/l15/rtl/noc3encoder.v b/piton/design/chip/tile/l15/rtl/noc3encoder.v
@@ -1,3 +1,4 @@
+// Modified by Barcelona Supercomputing Center on March 3rd, 2022
 /*
 Copyright (c) 2015 Princeton University
 All rights reserved.
@@ -231,6 +232,9 @@ begin
     msg_options_2[`MSG_CACHE_TYPE_] = msg_cache_type;
     msg_options_2[`MSG_SUBLINE_VECTOR_] = msg_subline_vector;
 
+    msg_options_3[`MSG_INI_X_] = coreid_x;
+    msg_options_3[`MSG_INI_Y_] = coreid_y;
+
     msg_options_4[`MSG_LAST_SUBLINE] = msg_last_subline || (l15_noc3encoder_req_type == `L15_NOC3_REQTYPE_ICACHE_INVAL_ACK);
     msg_options_4[`MSG_SUBLINE_ID] = l15_noc3encoder_req_sequenceid;
     // does not need to specify cache line state

diff --git a/piton/design/chip/tile/l2/rtl/Flist.l2 b/piton/design/chip/tile/l2/rtl/Flist.l2
@@ -58,6 +58,7 @@ l2_config_regs.v
 l2_priority_encoder.v
 l2_broadcast_counter.v
 l2_broadcast_counter_wrap.v
+l2_to_mc.v
 
 sram_wrappers/sram_l2_tag.v
 sram_wrappers/sram_l2_data.v

diff --git a/piton/design/chip/tile/l2/rtl/l2.core b/piton/design/chip/tile/l2/rtl/l2.core
@@ -67,6 +67,7 @@ generate:
                           [l2_pipe2_dpath.v.pyv, l2_pipe2_dpath.tmp.v],      
                           [l2_priority_encoder.v.pyv, l2_priority_encoder.tmp.v], 
                           [l2_smc.v.pyv, l2_smc.tmp.v],              
+                          [l2_to_mc.v.pyv, l2_to_mc.v],             
                           [l2_state.v.pyv, l2_state.tmp.v]]            
 
 

diff --git a/piton/design/chip/tile/l2/rtl/l2.v b/piton/design/chip/tile/l2/rtl/l2.v
@@ -659,6 +659,12 @@ l2_pipe1 pipe1(
     .smc_tag_out            (smc_tag_out),
     `endif
 
+`ifdef PITON_EXTRA_MEMS
+    .chipid                 (chipid),
+    .coreid_x               (coreid_x),
+    .coreid_y               (coreid_y),
+`endif
+
     .data_clk_en            (data_clk_en_p1),
     .data_rdw_en            (data_rdw_en_p1),
     .data_addr              (data_addr_p1),

diff --git a/piton/design/chip/tile/l2/rtl/l2_decoder.v b/piton/design/chip/tile/l2/rtl/l2_decoder.v
@@ -1,3 +1,4 @@
+// Modified by Barcelona Supercomputing Center on March 3rd, 2022
 /*
 Copyright (c) 2015 Princeton University
 All rights reserved.
@@ -59,6 +60,8 @@ module l2_decoder(
     output reg [`MSG_SRC_CHIPID_WIDTH-1:0] msg_src_chipid,
     output reg [`MSG_SRC_X_WIDTH-1:0] msg_src_x,
     output reg [`MSG_SRC_Y_WIDTH-1:0] msg_src_y,
+    output reg [`MSG_SRC_X_WIDTH-1:0] msg_ini_x,
+    output reg [`MSG_SRC_Y_WIDTH-1:0] msg_ini_y,
     output reg [`MSG_SRC_FBITS_WIDTH-1:0] msg_src_fbits,
     output reg [`MSG_SDID_WIDTH-1:0] msg_sdid,
     output reg [`MSG_LSID_WIDTH-1:0] msg_lsid,
@@ -81,6 +84,8 @@ begin
     msg_src_chipid = msg_header[`MSG_SRC_CHIPID];
     msg_src_x = msg_header[`MSG_SRC_X];
     msg_src_y = msg_header[`MSG_SRC_Y];
+    msg_ini_x = msg_header[`MSG_INI_X];
+    msg_ini_y = msg_header[`MSG_INI_Y];
     msg_src_fbits = msg_header[`MSG_SRC_FBITS];
     msg_sdid = msg_header[`MSG_SDID];
     msg_lsid = msg_header[`MSG_LSID];