upstream/dynamips-0.2.7-RC3/mips_mts.c

/*
 * Cisco router simulation platform.
 * Copyright (c) 2005,2006 Christophe Fillot (cf@utc.fr)
 *
 * Template code for MTS.
 */

#define MTS_ENTRY  MTS_NAME(entry_t)
#define MTS_CACHE(cpu)  ( cpu->mts_u. MTS_NAME(cache) )

/* Forward declarations */
static forced_inline void *MTS_PROTO(access)(cpu_mips_t *cpu,m_uint64_t vaddr,
                                             u_int op_code,u_int op_size,
                                             u_int op_type,m_uint64_t *data,
                                             u_int *exc);

static fastcall int MTS_PROTO(translate)(cpu_mips_t *cpu,m_uint64_t vaddr,
                                         m_uint32_t *phys_page);

/* Initialize the MTS subsystem for the specified CPU */
int MTS_PROTO(init)(cpu_mips_t *cpu)
{
   size_t len;

   /* Initialize the cache entries to 0 (empty) */
   len = MTS_NAME_UP(HASH_SIZE) * sizeof(MTS_ENTRY);
   if (!(MTS_CACHE(cpu) = malloc(len)))
      return(-1);

   memset(MTS_CACHE(cpu),0xFF,len);
   cpu->mts_lookups = 0;
   cpu->mts_misses  = 0;
   return(0);
}

/* Free memory used by MTS */
void MTS_PROTO(shutdown)(cpu_mips_t *cpu)
{
   /* Free the cache itself */
   free(MTS_CACHE(cpu));
   MTS_CACHE(cpu) = NULL;
}

/* Show MTS detailed information (debugging only!) */
void MTS_PROTO(show_stats)(cpu_gen_t *gen_cpu)
{
   cpu_mips_t *cpu = CPU_MIPS64(gen_cpu);
#if DEBUG_MTS_MAP_VIRT
   MTS_ENTRY *entry;
   u_int i,count;
#endif

   printf("\nCPU%u: MTS%d statistics:\n",cpu->gen->id,MTS_ADDR_SIZE);

#if DEBUG_MTS_MAP_VIRT
   /* Valid hash entries */
   for(count=0,i=0;i<MTS_NAME_UP(HASH_SIZE);i++) {
      entry = &(MTS_CACHE(cpu)[i]);

      if (!(entry->gvpa & MTS_INV_ENTRY_MASK)) {
         printf("    %4u: vaddr=0x%8.8llx, paddr=0x%8.8llx, hpa=%p\n",
                i,(m_uint64_t)entry->gvpa,(m_uint64_t)entry->gppa,
                (void *)entry->hpa);
         count++;
      }
   }

   printf("   %u/%u valid hash entries.\n",count,MTS_NAME_UP(HASH_SIZE));
#endif

   printf("   Total lookups: %llu, misses: %llu, efficiency: %g%%\n",
          cpu->mts_lookups, cpu->mts_misses,
          100 - ((double)(cpu->mts_misses*100)/
                 (double)cpu->mts_lookups));
}

/* Invalidate the complete MTS cache */
void MTS_PROTO(invalidate_cache)(cpu_mips_t *cpu)
{
   size_t len;

   len = MTS_NAME_UP(HASH_SIZE) * sizeof(MTS_ENTRY);
   memset(MTS_CACHE(cpu),0xFF,len);
}

/* Invalidate partially the MTS cache, given a TLB entry index */
void MTS_PROTO(invalidate_tlb_entry)(cpu_mips_t *cpu,u_int tlb_index)
{
   MTS_PROTO(invalidate_cache)(cpu);
} 

/* 
 * MTS mapping.
 *
 * It is NOT inlined since it triggers a GCC bug on my config (x86, GCC 3.3.5)
 */
static no_inline MTS_ENTRY *
MTS_PROTO(map)(cpu_mips_t *cpu,u_int op_type,mts_map_t *map,
               MTS_ENTRY *entry,MTS_ENTRY *alt_entry)
{
   struct vdevice *dev;
   m_uint32_t offset;
   m_iptr_t host_ptr;
   int cow;

   if (!(dev = dev_lookup(cpu->vm,map->paddr,map->cached)))
      return NULL;

   if (dev->flags & VDEVICE_FLAG_SPARSE) {
      host_ptr = dev_sparse_get_host_addr(cpu->vm,dev,map->paddr,op_type,&cow);

      entry->gvpa  = map->vaddr;
      entry->gppa  = map->paddr;
      entry->hpa   = host_ptr;
      entry->flags = (cow) ? MTS_FLAG_COW : 0;
      return entry;
   }

   if (!dev->host_addr || (dev->flags & VDEVICE_FLAG_NO_MTS_MMAP)) {
      offset = map->paddr - dev->phys_addr;

      alt_entry->gvpa  = map->vaddr;
      alt_entry->gppa  = map->paddr;
      alt_entry->hpa   = (dev->id << MTS_DEVID_SHIFT) + offset;
      alt_entry->flags = MTS_FLAG_DEV;
      return alt_entry;
   }

   entry->gvpa  = map->vaddr;
   entry->gppa  = map->paddr;
   entry->hpa   = dev->host_addr + (map->paddr - dev->phys_addr);
   entry->flags = 0;

   return entry;
}

/* MTS lookup */
static void *MTS_PROTO(lookup)(cpu_mips_t *cpu,m_uint64_t vaddr)
{
   m_uint64_t data;
   u_int exc;
   return(MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_LOOKUP,4,MTS_READ,
                            &data,&exc));
}

/* === MIPS Memory Operations ============================================= */

/* LB: Load Byte */
fastcall u_int MTS_PROTO(lb)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
{
   m_uint64_t data;
   void *haddr;
   u_int exc;

   haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_LB,1,MTS_READ,&data,&exc);
   if (likely(haddr != NULL)) data = *(m_uint8_t *)haddr;
   if (likely(!exc)) cpu->gpr[reg] = sign_extend(data,8);
   return(exc);
}

/* LBU: Load Byte Unsigned */
fastcall u_int MTS_PROTO(lbu)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
{
   m_uint64_t data;
   void *haddr;
   u_int exc;

   haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_LBU,1,MTS_READ,&data,&exc);
   if (likely(haddr != NULL)) data = *(m_uint8_t *)haddr;
   if (likely(!exc)) cpu->gpr[reg] = data & 0xff;
   return(exc);
}

/* LH: Load Half-Word */
fastcall u_int MTS_PROTO(lh)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
{
   m_uint64_t data;
   void *haddr;
   u_int exc;

   haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_LH,2,MTS_READ,&data,&exc);
   if (likely(haddr != NULL)) data = vmtoh16(*(m_uint16_t *)haddr);
   if (likely(!exc)) cpu->gpr[reg] = sign_extend(data,16);
   return(exc);
}

/* LHU: Load Half-Word Unsigned */
fastcall u_int MTS_PROTO(lhu)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
{
   m_uint64_t data;
   void *haddr;
   u_int exc;

   haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_LHU,2,MTS_READ,&data,&exc);
   if (likely(haddr != NULL)) data = vmtoh16(*(m_uint16_t *)haddr);
   if (likely(!exc)) cpu->gpr[reg] = data & 0xffff;
   return(exc);
}

/* LW: Load Word */
fastcall u_int MTS_PROTO(lw)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
{
   m_uint64_t data;
   void *haddr;
   u_int exc;

   haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_LW,4,MTS_READ,&data,&exc);
   if (likely(haddr != NULL)) data = vmtoh32(*(m_uint32_t *)haddr);
   if (likely(!exc)) cpu->gpr[reg] = sign_extend(data,32);
   return(exc);
}

/* LWU: Load Word Unsigned */
fastcall u_int MTS_PROTO(lwu)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
{
   m_uint64_t data;
   void *haddr;
   u_int exc;

   haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_LWU,4,MTS_READ,&data,&exc);
   if (likely(haddr != NULL)) data = vmtoh32(*(m_uint32_t *)haddr);
   if (likely(!exc)) cpu->gpr[reg] = data & 0xffffffff;
   return(exc);
}

/* LD: Load Double-Word */
fastcall u_int MTS_PROTO(ld)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
{
   m_uint64_t data;
   void *haddr;
   u_int exc;

   haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_LD,8,MTS_READ,&data,&exc);
   if (likely(haddr != NULL)) data = vmtoh64(*(m_uint64_t *)haddr);
   if (likely(!exc)) cpu->gpr[reg] = data;
   return(exc);
}

/* SB: Store Byte */
fastcall u_int MTS_PROTO(sb)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
{
   m_uint64_t data;
   void *haddr;
   u_int exc;

   data = cpu->gpr[reg] & 0xff;
   haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_SB,1,MTS_WRITE,&data,&exc);
   if (likely(haddr != NULL)) *(m_uint8_t *)haddr = data;
   return(exc);
}

/* SH: Store Half-Word */
fastcall u_int MTS_PROTO(sh)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
{
   m_uint64_t data;
   void *haddr;
   u_int exc;

   data = cpu->gpr[reg] & 0xffff;
   haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_SH,2,MTS_WRITE,&data,&exc);
   if (likely(haddr != NULL)) *(m_uint16_t *)haddr = htovm16(data);
   return(exc);
}

/* SW: Store Word */
fastcall u_int MTS_PROTO(sw)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
{
   m_uint64_t data;
   void *haddr;
   u_int exc;

   data = cpu->gpr[reg] & 0xffffffff;
   haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_SW,4,MTS_WRITE,&data,&exc);
   if (likely(haddr != NULL)) *(m_uint32_t *)haddr = htovm32(data);
   return(exc);
}

/* SD: Store Double-Word */
fastcall u_int MTS_PROTO(sd)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
{
   m_uint64_t data;
   void *haddr;
   u_int exc;

   data = cpu->gpr[reg];
   haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_SD,8,MTS_WRITE,&data,&exc);
   if (likely(haddr != NULL)) *(m_uint64_t *)haddr = htovm64(data);
   return(exc);
}

/* LDC1: Load Double-Word To Coprocessor 1 */
fastcall u_int MTS_PROTO(ldc1)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
{
   m_uint64_t data;
   void *haddr;
   u_int exc;

   haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_LDC1,8,MTS_READ,&data,&exc);
   if (likely(haddr != NULL)) data = vmtoh64(*(m_uint64_t *)haddr);
   if (likely(!exc)) cpu->fpu.reg[reg] = data;
   return(exc);
}

/* LWL: Load Word Left */
fastcall u_int MTS_PROTO(lwl)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
{
   m_uint64_t r_mask,naddr;
   m_uint64_t data;
   u_int m_shift;
   void *haddr;
   u_int exc;

   naddr = vaddr & ~(0x03);
   haddr = MTS_PROTO(access)(cpu,naddr,MIPS_MEMOP_LWL,4,MTS_READ,&data,&exc);

   if (likely(haddr != NULL))
      data = vmtoh32(*(m_uint32_t *)haddr);

   if (likely(!exc)) {
      m_shift = (vaddr & 0x03) << 3;
      r_mask = (1ULL << m_shift) - 1;
      data <<= m_shift;

      cpu->gpr[reg] &= r_mask;
      cpu->gpr[reg] |= data;
      cpu->gpr[reg] = sign_extend(cpu->gpr[reg],32);
   }
   return(exc);
}

/* LWR: Load Word Right */
fastcall u_int MTS_PROTO(lwr)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
{
   m_uint64_t r_mask,naddr;
   m_uint64_t data;
   u_int m_shift;
   void *haddr;
   u_int exc;

   naddr = vaddr & ~(0x03);
   haddr = MTS_PROTO(access)(cpu,naddr,MIPS_MEMOP_LWR,4,MTS_READ,&data,&exc);

   if (likely(haddr != NULL))
      data = vmtoh32(*(m_uint32_t *)haddr);

   if (likely(!exc)) {
      m_shift = ((vaddr & 0x03) + 1) << 3;
      r_mask = (1ULL << m_shift) - 1;

      data = sign_extend(data >> (32 - m_shift),32);
      r_mask = sign_extend(r_mask,32);

      cpu->gpr[reg] &= ~r_mask;
      cpu->gpr[reg] |= data;
   }
   return(exc);
}

/* LDL: Load Double-Word Left */
fastcall u_int MTS_PROTO(ldl)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
{
   m_uint64_t r_mask,naddr;
   m_uint64_t data;
   u_int m_shift;
   void *haddr;
   u_int exc;

   naddr = vaddr & ~(0x07);
   haddr = MTS_PROTO(access)(cpu,naddr,MIPS_MEMOP_LDL,8,MTS_READ,&data,&exc);

   if (likely(haddr != NULL))
      data = vmtoh64(*(m_uint64_t *)haddr);

   if (likely(!exc)) {
      m_shift = (vaddr & 0x07) << 3;
      r_mask = (1ULL << m_shift) - 1;
      data <<= m_shift;

      cpu->gpr[reg] &= r_mask;
      cpu->gpr[reg] |= data;
   }
   return(exc);
}

/* LDR: Load Double-Word Right */
fastcall u_int MTS_PROTO(ldr)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
{
   m_uint64_t r_mask,naddr;
   m_uint64_t data;
   u_int m_shift;
   void *haddr;
   u_int exc;

   naddr = vaddr & ~(0x07);
   haddr = MTS_PROTO(access)(cpu,naddr,MIPS_MEMOP_LDR,8,MTS_READ,&data,&exc);

   if (likely(haddr != NULL))
      data = vmtoh64(*(m_uint64_t *)haddr);

   if (likely(!exc)) {
      m_shift = ((vaddr & 0x07) + 1) << 3;
      r_mask = (1ULL << m_shift) - 1;
      data >>= (64 - m_shift);

      cpu->gpr[reg] &= ~r_mask;
      cpu->gpr[reg] |= data;
   }
   return(exc);
}

/* SWL: Store Word Left */
fastcall u_int MTS_PROTO(swl)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
{
   m_uint64_t d_mask,naddr;
   m_uint64_t data;
   u_int r_shift;
   void *haddr;
   u_int exc;

   naddr = vaddr & ~(0x03ULL);
   haddr = MTS_PROTO(access)(cpu,naddr,MIPS_MEMOP_SWL,4,MTS_READ,&data,&exc);
   if (unlikely(exc)) return(exc);

   if (likely(haddr != NULL))
      data = vmtoh32(*(m_uint32_t *)haddr);

   r_shift = (vaddr & 0x03) << 3;
   d_mask = 0xffffffff >> r_shift;

   data &= ~d_mask;
   data |= (cpu->gpr[reg] & 0xffffffff) >> r_shift;

   haddr = MTS_PROTO(access)(cpu,naddr,MIPS_MEMOP_SWL,4,MTS_WRITE,&data,&exc);
   if (likely(haddr != NULL)) *(m_uint32_t *)haddr = htovm32(data);
   return(exc);
}

/* SWR: Store Word Right */
fastcall u_int MTS_PROTO(swr)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
{
   m_uint64_t d_mask,naddr;
   m_uint64_t data;
   u_int r_shift;
   void *haddr;
   u_int exc;

   naddr = vaddr & ~(0x03);
   haddr = MTS_PROTO(access)(cpu,naddr,MIPS_MEMOP_SWR,4,MTS_READ,&data,&exc);
   if (unlikely(exc)) return(exc);

   if (likely(haddr != NULL))
      data = vmtoh32(*(m_uint32_t *)haddr);

   r_shift = ((vaddr & 0x03) + 1) << 3;
   d_mask = 0xffffffff >> r_shift;

   data &= d_mask;
   data |= (cpu->gpr[reg] << (32 - r_shift)) & 0xffffffff;

   haddr = MTS_PROTO(access)(cpu,naddr,MIPS_MEMOP_SWR,4,MTS_WRITE,&data,&exc);
   if (likely(haddr != NULL)) *(m_uint32_t *)haddr = htovm32(data);
   return(exc);
}

/* SDL: Store Double-Word Left */
fastcall u_int MTS_PROTO(sdl)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
{
   m_uint64_t d_mask,naddr;
   m_uint64_t data;
   u_int r_shift;
   void *haddr;
   u_int exc;

   naddr = vaddr & ~(0x07);
   haddr = MTS_PROTO(access)(cpu,naddr,MIPS_MEMOP_SDL,8,MTS_READ,&data,&exc);
   if (unlikely(exc)) return(exc);

   if (likely(haddr != NULL))
      data = vmtoh64(*(m_uint64_t *)haddr);

   r_shift = (vaddr & 0x07) << 3;
   d_mask = 0xffffffffffffffffULL >> r_shift;

   data &= ~d_mask;
   data |= cpu->gpr[reg] >> r_shift;

   haddr = MTS_PROTO(access)(cpu,naddr,MIPS_MEMOP_SDL,8,MTS_WRITE,&data,&exc);
   if (likely(haddr != NULL)) *(m_uint64_t *)haddr = htovm64(data);
   return(exc);
}

/* SDR: Store Double-Word Right */
fastcall u_int MTS_PROTO(sdr)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
{
   m_uint64_t d_mask,naddr;
   m_uint64_t data;
   u_int r_shift;
   void *haddr;
   u_int exc;

   naddr = vaddr & ~(0x07);
   haddr = MTS_PROTO(access)(cpu,naddr,MIPS_MEMOP_SDR,8,MTS_READ,&data,&exc);
   if (unlikely(exc)) return(exc);

   if (likely(haddr != NULL))
      data = vmtoh64(*(m_uint64_t *)haddr);

   r_shift = ((vaddr & 0x07) + 1) << 3;
   d_mask = 0xffffffffffffffffULL >> r_shift;

   data &= d_mask;
   data |= cpu->gpr[reg] << (64 - r_shift);

   haddr = MTS_PROTO(access)(cpu,naddr,MIPS_MEMOP_SDR,8,MTS_WRITE,&data,&exc);
   if (likely(haddr != NULL)) *(m_uint64_t *)haddr = htovm64(data);
   return(exc);
}

/* LL: Load Linked */
fastcall u_int MTS_PROTO(ll)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
{
   m_uint64_t data;
   void *haddr;
   u_int exc;

   haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_LL,4,MTS_READ,&data,&exc);
   if (likely(haddr != NULL)) data = vmtoh32(*(m_uint32_t *)haddr);

   if (likely(!exc)) {
      cpu->gpr[reg] = sign_extend(data,32);
      cpu->ll_bit = 1;
   }

   return(exc);
}

/* SC: Store Conditional */
fastcall u_int MTS_PROTO(sc)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
{
   m_uint64_t data;
   void *haddr;
   u_int exc = 0;

   if (cpu->ll_bit) {
      data = cpu->gpr[reg] & 0xffffffff;
      haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_SC,4,MTS_WRITE,
                                &data,&exc);
      if (likely(haddr != NULL)) *(m_uint32_t *)haddr = htovm32(data);
   }

   if (likely(!exc))
      cpu->gpr[reg] = cpu->ll_bit;
   return(exc);
}

/* SDC1: Store Double-Word from Coprocessor 1 */
fastcall u_int MTS_PROTO(sdc1)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
{
   m_uint64_t data;
   void *haddr;
   u_int exc;

   data = cpu->fpu.reg[reg];
   haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_SDC1,8,MTS_WRITE,
                             &data,&exc);
   if (likely(haddr != NULL)) *(m_uint64_t *)haddr = htovm64(data);
   return(exc);
}

/* CACHE: Cache operation */
fastcall u_int MTS_PROTO(cache)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int op)
{
   mips64_jit_tcb_t *block;
   m_uint32_t phys_page;

#if DEBUG_CACHE
   cpu_log(cpu->gen,
           "MTS","CACHE: PC=0x%llx, vaddr=0x%llx, cache=%u, code=%u\n",
           cpu->pc, vaddr, op & 0x3, op >> 2);
#endif

   if (!cpu->translate(cpu,vaddr,&phys_page)) {
      if ((phys_page < 1048576) && cpu->exec_phys_map) {
         block = cpu->exec_phys_map[phys_page];

         if (block) {
            if ((cpu->pc < block->start_pc) || 
                ((cpu->pc - block->start_pc) >= MIPS_MIN_PAGE_SIZE))
            {
#if DEBUG_CACHE
               cpu_log(cpu->gen,"MTS",
                       "CACHE: removing compiled page at 0x%llx, pc=0x%llx\n",
                       block->start_pc,cpu->pc);
#endif
               cpu->exec_phys_map[phys_page] = NULL;
               mips64_jit_tcb_free(cpu,block,TRUE);
            }
            else
            {
#if DEBUG_CACHE
               cpu_log(cpu->gen,"MTS",
                       "CACHE: trying to remove page 0x%llx with pc=0x%llx\n",
                       block->start_pc,cpu->pc);
#endif
            }
         }
      }
   }

   return(0);
}

/* === MTS Cache Management ============================================= */

/* MTS map/unmap/rebuild "API" functions */
void MTS_PROTO(api_map)(cpu_mips_t *cpu,m_uint64_t vaddr,m_uint64_t paddr,
                        m_uint32_t len,int cache_access,int tlb_index)
{
   /* nothing to do, the cache will be filled on-the-fly */
}

void MTS_PROTO(api_unmap)(cpu_mips_t *cpu,m_uint64_t vaddr,m_uint32_t len,
                          m_uint32_t val,int tlb_index)
{
   /* Invalidate the TLB entry or the full cache if no index is specified */
   if (tlb_index != -1)
      MTS_PROTO(invalidate_tlb_entry)(cpu,tlb_index);
   else
      MTS_PROTO(invalidate_cache)(cpu);
}

void MTS_PROTO(api_rebuild)(cpu_gen_t *cpu)
{
   MTS_PROTO(invalidate_cache)(CPU_MIPS64(cpu));
}

/* ======================================================================== */

/* Initialize memory access vectors */
void MTS_PROTO(init_memop_vectors)(cpu_mips_t *cpu)
{
   /* XXX TODO:
    *  - LD/SD forbidden in Supervisor/User modes with 32-bit addresses.
    */

   cpu->addr_mode = MTS_ADDR_SIZE;

   /* API vectors */
   cpu->mts_map     = MTS_PROTO(api_map);
   cpu->mts_unmap   = MTS_PROTO(api_unmap);

   /* Memory lookup operation */
   cpu->mem_op_lookup = MTS_PROTO(lookup);

   /* Translation operation */
   cpu->translate = MTS_PROTO(translate);

   /* Shutdown operation */
   cpu->mts_shutdown = MTS_PROTO(shutdown);

   /* Rebuild MTS data structures */
   cpu->gen->mts_rebuild = MTS_PROTO(api_rebuild);

   /* Show statistics */
   cpu->gen->mts_show_stats = MTS_PROTO(show_stats);

   /* Load Operations */
   cpu->mem_op_fn[MIPS_MEMOP_LB] = MTS_PROTO(lb);
   cpu->mem_op_fn[MIPS_MEMOP_LBU] = MTS_PROTO(lbu);
   cpu->mem_op_fn[MIPS_MEMOP_LH] = MTS_PROTO(lh);
   cpu->mem_op_fn[MIPS_MEMOP_LHU] = MTS_PROTO(lhu);
   cpu->mem_op_fn[MIPS_MEMOP_LW] = MTS_PROTO(lw);
   cpu->mem_op_fn[MIPS_MEMOP_LWU] = MTS_PROTO(lwu);
   cpu->mem_op_fn[MIPS_MEMOP_LD] = MTS_PROTO(ld);
   cpu->mem_op_fn[MIPS_MEMOP_LDL] = MTS_PROTO(ldl);
   cpu->mem_op_fn[MIPS_MEMOP_LDR] = MTS_PROTO(ldr);

   /* Store Operations */
   cpu->mem_op_fn[MIPS_MEMOP_SB] = MTS_PROTO(sb);
   cpu->mem_op_fn[MIPS_MEMOP_SH] = MTS_PROTO(sh);
   cpu->mem_op_fn[MIPS_MEMOP_SW] = MTS_PROTO(sw);
   cpu->mem_op_fn[MIPS_MEMOP_SD] = MTS_PROTO(sd);

   /* Load Left/Right operations */
   cpu->mem_op_fn[MIPS_MEMOP_LWL] = MTS_PROTO(lwl);
   cpu->mem_op_fn[MIPS_MEMOP_LWR] = MTS_PROTO(lwr);
   cpu->mem_op_fn[MIPS_MEMOP_LDL] = MTS_PROTO(ldl);
   cpu->mem_op_fn[MIPS_MEMOP_LDR] = MTS_PROTO(ldr);

   /* Store Left/Right operations */
   cpu->mem_op_fn[MIPS_MEMOP_SWL] = MTS_PROTO(swl);
   cpu->mem_op_fn[MIPS_MEMOP_SWR] = MTS_PROTO(swr);
   cpu->mem_op_fn[MIPS_MEMOP_SDL] = MTS_PROTO(sdl);
   cpu->mem_op_fn[MIPS_MEMOP_SDR] = MTS_PROTO(sdr);

   /* LL/SC - Load Linked / Store Conditional */
   cpu->mem_op_fn[MIPS_MEMOP_LL] = MTS_PROTO(ll);
   cpu->mem_op_fn[MIPS_MEMOP_SC] = MTS_PROTO(sc);

   /* Coprocessor 1 memory access functions */
   cpu->mem_op_fn[MIPS_MEMOP_LDC1] = MTS_PROTO(ldc1);
   cpu->mem_op_fn[MIPS_MEMOP_SDC1] = MTS_PROTO(sdc1);

   /* Cache Operation */
   cpu->mem_op_fn[MIPS_MEMOP_CACHE] = MTS_PROTO(cache);
}

#undef MTS_ADDR_SIZE
#undef MTS_NAME
#undef MTS_NAME_UP
#undef MTS_PROTO
#undef MTS_PROTO_UP
#undef MTS_ENTRY
#undef MTS_CHUNK
1	dpavlin	4	/*
2			* Cisco router simulation platform.
3			* Copyright (c) 2005,2006 Christophe Fillot (cf@utc.fr)
4			*
5			* Template code for MTS.
6			*/
7
8	dpavlin	7	#define MTS_ENTRY MTS_NAME(entry_t)
9			#define MTS_CACHE(cpu) ( cpu->mts_u. MTS_NAME(cache) )
10	dpavlin	4
11			/* Forward declarations */
12			static forced_inline void MTS_PROTO(access)(cpu_mips_t cpu,m_uint64_t vaddr,
13			u_int op_code,u_int op_size,
14			u_int op_type,m_uint64_t *data,
15			u_int *exc);
16
17			static fastcall int MTS_PROTO(translate)(cpu_mips_t *cpu,m_uint64_t vaddr,
18			m_uint32_t *phys_page);
19
20			/* Initialize the MTS subsystem for the specified CPU */
21			int MTS_PROTO(init)(cpu_mips_t *cpu)
22			{
23			size_t len;
24
25			/* Initialize the cache entries to 0 (empty) */
26	dpavlin	7	len = MTS_NAME_UP(HASH_SIZE) * sizeof(MTS_ENTRY);
27			if (!(MTS_CACHE(cpu) = malloc(len)))
28	dpavlin	4	return(-1);
29
30	dpavlin	7	memset(MTS_CACHE(cpu),0xFF,len);
31	dpavlin	4	cpu->mts_lookups = 0;
32			cpu->mts_misses = 0;
33			return(0);
34			}
35
36			/* Free memory used by MTS */
37			void MTS_PROTO(shutdown)(cpu_mips_t *cpu)
38			{
39			/* Free the cache itself */
40	dpavlin	7	free(MTS_CACHE(cpu));
41			MTS_CACHE(cpu) = NULL;
42	dpavlin	4	}
43
44			/* Show MTS detailed information (debugging only!) */
45	dpavlin	7	void MTS_PROTO(show_stats)(cpu_gen_t *gen_cpu)
46	dpavlin	4	{
47	dpavlin	7	cpu_mips_t *cpu = CPU_MIPS64(gen_cpu);
48	dpavlin	4	#if DEBUG_MTS_MAP_VIRT
49			MTS_ENTRY *entry;
50	dpavlin	7	u_int i,count;
51	dpavlin	4	#endif
52
53	dpavlin	7	printf("\nCPU%u: MTS%d statistics:\n",cpu->gen->id,MTS_ADDR_SIZE);
54	dpavlin	4
55			#if DEBUG_MTS_MAP_VIRT
56			/* Valid hash entries */
57	dpavlin	7	for(count=0,i=0;i<MTS_NAME_UP(HASH_SIZE);i++) {
58			entry = &(MTS_CACHE(cpu)[i]);
59
60			if (!(entry->gvpa & MTS_INV_ENTRY_MASK)) {
61			printf(" %4u: vaddr=0x%8.8llx, paddr=0x%8.8llx, hpa=%p\n",
62			i,(m_uint64_t)entry->gvpa,(m_uint64_t)entry->gppa,
63			(void *)entry->hpa);
64	dpavlin	4	count++;
65			}
66			}
67
68	dpavlin	7	printf(" %u/%u valid hash entries.\n",count,MTS_NAME_UP(HASH_SIZE));
69	dpavlin	4	#endif
70
71	dpavlin	7	printf(" Total lookups: %llu, misses: %llu, efficiency: %g%%\n",
72			cpu->mts_lookups, cpu->mts_misses,
73			100 - ((double)(cpu->mts_misses*100)/
74			(double)cpu->mts_lookups));
75	dpavlin	4	}
76
77			/* Invalidate the complete MTS cache */
78			void MTS_PROTO(invalidate_cache)(cpu_mips_t *cpu)
79			{
80			size_t len;
81
82	dpavlin	7	len = MTS_NAME_UP(HASH_SIZE) * sizeof(MTS_ENTRY);
83			memset(MTS_CACHE(cpu),0xFF,len);
84	dpavlin	4	}
85
86			/* Invalidate partially the MTS cache, given a TLB entry index */
87			void MTS_PROTO(invalidate_tlb_entry)(cpu_mips_t *cpu,u_int tlb_index)
88			{
89	dpavlin	7	MTS_PROTO(invalidate_cache)(cpu);
90	dpavlin	4	}
91
92			/*
93			* MTS mapping.
94			*
95			* It is NOT inlined since it triggers a GCC bug on my config (x86, GCC 3.3.5)
96			*/
97	dpavlin	7	static no_inline MTS_ENTRY *
98			MTS_PROTO(map)(cpu_mips_t cpu,u_int op_type,mts_map_t map,
99			MTS_ENTRY entry,MTS_ENTRY alt_entry)
100	dpavlin	4	{
101			struct vdevice *dev;
102	dpavlin	7	m_uint32_t offset;
103			m_iptr_t host_ptr;
104			int cow;
105	dpavlin	4
106	dpavlin	7	if (!(dev = dev_lookup(cpu->vm,map->paddr,map->cached)))
107			return NULL;
108	dpavlin	4
109	dpavlin	7	if (dev->flags & VDEVICE_FLAG_SPARSE) {
110			host_ptr = dev_sparse_get_host_addr(cpu->vm,dev,map->paddr,op_type,&cow);
111	dpavlin	4
112	dpavlin	7	entry->gvpa = map->vaddr;
113			entry->gppa = map->paddr;
114			entry->hpa = host_ptr;
115			entry->flags = (cow) ? MTS_FLAG_COW : 0;
116			return entry;
117	dpavlin	4	}
118
119	dpavlin	7	if (!dev->host_addr \|\| (dev->flags & VDEVICE_FLAG_NO_MTS_MMAP)) {
120			offset = map->paddr - dev->phys_addr;
121	dpavlin	4
122	dpavlin	7	alt_entry->gvpa = map->vaddr;
123			alt_entry->gppa = map->paddr;
124			alt_entry->hpa = (dev->id << MTS_DEVID_SHIFT) + offset;
125			alt_entry->flags = MTS_FLAG_DEV;
126			return alt_entry;
127			}
128
129			entry->gvpa = map->vaddr;
130			entry->gppa = map->paddr;
131			entry->hpa = dev->host_addr + (map->paddr - dev->phys_addr);
132			entry->flags = 0;
133
134			return entry;
135	dpavlin	4	}
136
137			/* MTS lookup */
138			static void MTS_PROTO(lookup)(cpu_mips_t cpu,m_uint64_t vaddr)
139			{
140			m_uint64_t data;
141			u_int exc;
142			return(MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_LOOKUP,4,MTS_READ,
143			&data,&exc));
144			}
145
146			/* === MIPS Memory Operations ============================================= */
147
148			/* LB: Load Byte */
149			fastcall u_int MTS_PROTO(lb)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
150			{
151			m_uint64_t data;
152			void *haddr;
153			u_int exc;
154
155			haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_LB,1,MTS_READ,&data,&exc);
156			if (likely(haddr != NULL)) data = (m_uint8_t )haddr;
157			if (likely(!exc)) cpu->gpr[reg] = sign_extend(data,8);
158			return(exc);
159			}
160
161			/* LBU: Load Byte Unsigned */
162			fastcall u_int MTS_PROTO(lbu)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
163			{
164			m_uint64_t data;
165			void *haddr;
166			u_int exc;
167
168			haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_LBU,1,MTS_READ,&data,&exc);
169			if (likely(haddr != NULL)) data = (m_uint8_t )haddr;
170			if (likely(!exc)) cpu->gpr[reg] = data & 0xff;
171			return(exc);
172			}
173
174			/* LH: Load Half-Word */
175			fastcall u_int MTS_PROTO(lh)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
176			{
177			m_uint64_t data;
178			void *haddr;
179			u_int exc;
180
181			haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_LH,2,MTS_READ,&data,&exc);
182			if (likely(haddr != NULL)) data = vmtoh16((m_uint16_t )haddr);
183			if (likely(!exc)) cpu->gpr[reg] = sign_extend(data,16);
184			return(exc);
185			}
186
187			/* LHU: Load Half-Word Unsigned */
188			fastcall u_int MTS_PROTO(lhu)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
189			{
190			m_uint64_t data;
191			void *haddr;
192			u_int exc;
193
194			haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_LHU,2,MTS_READ,&data,&exc);
195			if (likely(haddr != NULL)) data = vmtoh16((m_uint16_t )haddr);
196			if (likely(!exc)) cpu->gpr[reg] = data & 0xffff;
197			return(exc);
198			}
199
200			/* LW: Load Word */
201			fastcall u_int MTS_PROTO(lw)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
202			{
203			m_uint64_t data;
204			void *haddr;
205			u_int exc;
206
207			haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_LW,4,MTS_READ,&data,&exc);
208			if (likely(haddr != NULL)) data = vmtoh32((m_uint32_t )haddr);
209			if (likely(!exc)) cpu->gpr[reg] = sign_extend(data,32);
210			return(exc);
211			}
212
213			/* LWU: Load Word Unsigned */
214			fastcall u_int MTS_PROTO(lwu)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
215			{
216			m_uint64_t data;
217			void *haddr;
218			u_int exc;
219
220			haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_LWU,4,MTS_READ,&data,&exc);
221			if (likely(haddr != NULL)) data = vmtoh32((m_uint32_t )haddr);
222			if (likely(!exc)) cpu->gpr[reg] = data & 0xffffffff;
223			return(exc);
224			}
225
226			/* LD: Load Double-Word */
227			fastcall u_int MTS_PROTO(ld)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
228			{
229			m_uint64_t data;
230			void *haddr;
231			u_int exc;
232
233			haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_LD,8,MTS_READ,&data,&exc);
234			if (likely(haddr != NULL)) data = vmtoh64((m_uint64_t )haddr);
235			if (likely(!exc)) cpu->gpr[reg] = data;
236			return(exc);
237			}
238
239			/* SB: Store Byte */
240			fastcall u_int MTS_PROTO(sb)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
241			{
242			m_uint64_t data;
243			void *haddr;
244			u_int exc;
245
246			data = cpu->gpr[reg] & 0xff;
247			haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_SB,1,MTS_WRITE,&data,&exc);
248			if (likely(haddr != NULL)) (m_uint8_t )haddr = data;
249			return(exc);
250			}
251
252			/* SH: Store Half-Word */
253			fastcall u_int MTS_PROTO(sh)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
254			{
255			m_uint64_t data;
256			void *haddr;
257			u_int exc;
258
259			data = cpu->gpr[reg] & 0xffff;
260			haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_SH,2,MTS_WRITE,&data,&exc);
261			if (likely(haddr != NULL)) (m_uint16_t )haddr = htovm16(data);
262			return(exc);
263			}
264
265			/* SW: Store Word */
266			fastcall u_int MTS_PROTO(sw)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
267			{
268			m_uint64_t data;
269			void *haddr;
270			u_int exc;
271
272			data = cpu->gpr[reg] & 0xffffffff;
273			haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_SW,4,MTS_WRITE,&data,&exc);
274			if (likely(haddr != NULL)) (m_uint32_t )haddr = htovm32(data);
275			return(exc);
276			}
277
278			/* SD: Store Double-Word */
279			fastcall u_int MTS_PROTO(sd)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
280			{
281			m_uint64_t data;
282			void *haddr;
283			u_int exc;
284
285			data = cpu->gpr[reg];
286			haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_SD,8,MTS_WRITE,&data,&exc);
287			if (likely(haddr != NULL)) (m_uint64_t )haddr = htovm64(data);
288			return(exc);
289			}
290
291			/* LDC1: Load Double-Word To Coprocessor 1 */
292			fastcall u_int MTS_PROTO(ldc1)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
293			{
294			m_uint64_t data;
295			void *haddr;
296			u_int exc;
297
298			haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_LDC1,8,MTS_READ,&data,&exc);
299			if (likely(haddr != NULL)) data = vmtoh64((m_uint64_t )haddr);
300			if (likely(!exc)) cpu->fpu.reg[reg] = data;
301			return(exc);
302			}
303
304			/* LWL: Load Word Left */
305			fastcall u_int MTS_PROTO(lwl)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
306			{
307			m_uint64_t r_mask,naddr;
308			m_uint64_t data;
309			u_int m_shift;
310			void *haddr;
311			u_int exc;
312
313			naddr = vaddr & ~(0x03);
314			haddr = MTS_PROTO(access)(cpu,naddr,MIPS_MEMOP_LWL,4,MTS_READ,&data,&exc);
315
316			if (likely(haddr != NULL))
317			data = vmtoh32((m_uint32_t )haddr);
318
319			if (likely(!exc)) {
320			m_shift = (vaddr & 0x03) << 3;
321			r_mask = (1ULL << m_shift) - 1;
322			data <<= m_shift;
323
324			cpu->gpr[reg] &= r_mask;
325			cpu->gpr[reg] \|= data;
326			cpu->gpr[reg] = sign_extend(cpu->gpr[reg],32);
327			}
328			return(exc);
329			}
330
331			/* LWR: Load Word Right */
332			fastcall u_int MTS_PROTO(lwr)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
333			{
334			m_uint64_t r_mask,naddr;
335			m_uint64_t data;
336			u_int m_shift;
337			void *haddr;
338			u_int exc;
339
340			naddr = vaddr & ~(0x03);
341			haddr = MTS_PROTO(access)(cpu,naddr,MIPS_MEMOP_LWR,4,MTS_READ,&data,&exc);
342
343			if (likely(haddr != NULL))
344			data = vmtoh32((m_uint32_t )haddr);
345
346			if (likely(!exc)) {
347			m_shift = ((vaddr & 0x03) + 1) << 3;
348			r_mask = (1ULL << m_shift) - 1;
349
350			data = sign_extend(data >> (32 - m_shift),32);
351			r_mask = sign_extend(r_mask,32);
352
353			cpu->gpr[reg] &= ~r_mask;
354			cpu->gpr[reg] \|= data;
355			}
356			return(exc);
357			}
358
359			/* LDL: Load Double-Word Left */
360			fastcall u_int MTS_PROTO(ldl)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
361			{
362			m_uint64_t r_mask,naddr;
363			m_uint64_t data;
364			u_int m_shift;
365			void *haddr;
366			u_int exc;
367
368			naddr = vaddr & ~(0x07);
369			haddr = MTS_PROTO(access)(cpu,naddr,MIPS_MEMOP_LDL,8,MTS_READ,&data,&exc);
370
371			if (likely(haddr != NULL))
372			data = vmtoh64((m_uint64_t )haddr);
373
374			if (likely(!exc)) {
375			m_shift = (vaddr & 0x07) << 3;
376			r_mask = (1ULL << m_shift) - 1;
377			data <<= m_shift;
378
379			cpu->gpr[reg] &= r_mask;
380			cpu->gpr[reg] \|= data;
381			}
382			return(exc);
383			}
384
385			/* LDR: Load Double-Word Right */
386			fastcall u_int MTS_PROTO(ldr)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
387			{
388			m_uint64_t r_mask,naddr;
389			m_uint64_t data;
390			u_int m_shift;
391			void *haddr;
392			u_int exc;
393
394			naddr = vaddr & ~(0x07);
395			haddr = MTS_PROTO(access)(cpu,naddr,MIPS_MEMOP_LDR,8,MTS_READ,&data,&exc);
396
397			if (likely(haddr != NULL))
398			data = vmtoh64((m_uint64_t )haddr);
399
400			if (likely(!exc)) {
401			m_shift = ((vaddr & 0x07) + 1) << 3;
402			r_mask = (1ULL << m_shift) - 1;
403			data >>= (64 - m_shift);
404
405			cpu->gpr[reg] &= ~r_mask;
406			cpu->gpr[reg] \|= data;
407			}
408			return(exc);
409			}
410
411			/* SWL: Store Word Left */
412			fastcall u_int MTS_PROTO(swl)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
413			{
414			m_uint64_t d_mask,naddr;
415			m_uint64_t data;
416			u_int r_shift;
417			void *haddr;
418			u_int exc;
419
420			naddr = vaddr & ~(0x03ULL);
421			haddr = MTS_PROTO(access)(cpu,naddr,MIPS_MEMOP_SWL,4,MTS_READ,&data,&exc);
422			if (unlikely(exc)) return(exc);
423
424			if (likely(haddr != NULL))
425			data = vmtoh32((m_uint32_t )haddr);
426
427			r_shift = (vaddr & 0x03) << 3;
428			d_mask = 0xffffffff >> r_shift;
429
430			data &= ~d_mask;
431			data \|= (cpu->gpr[reg] & 0xffffffff) >> r_shift;
432
433			haddr = MTS_PROTO(access)(cpu,naddr,MIPS_MEMOP_SWL,4,MTS_WRITE,&data,&exc);
434			if (likely(haddr != NULL)) (m_uint32_t )haddr = htovm32(data);
435			return(exc);
436			}
437
438			/* SWR: Store Word Right */
439			fastcall u_int MTS_PROTO(swr)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
440			{
441			m_uint64_t d_mask,naddr;
442			m_uint64_t data;
443			u_int r_shift;
444			void *haddr;
445			u_int exc;
446
447			naddr = vaddr & ~(0x03);
448			haddr = MTS_PROTO(access)(cpu,naddr,MIPS_MEMOP_SWR,4,MTS_READ,&data,&exc);
449			if (unlikely(exc)) return(exc);
450
451			if (likely(haddr != NULL))
452			data = vmtoh32((m_uint32_t )haddr);
453
454			r_shift = ((vaddr & 0x03) + 1) << 3;
455			d_mask = 0xffffffff >> r_shift;
456
457			data &= d_mask;
458			data \|= (cpu->gpr[reg] << (32 - r_shift)) & 0xffffffff;
459
460			haddr = MTS_PROTO(access)(cpu,naddr,MIPS_MEMOP_SWR,4,MTS_WRITE,&data,&exc);
461			if (likely(haddr != NULL)) (m_uint32_t )haddr = htovm32(data);
462			return(exc);
463			}
464
465			/* SDL: Store Double-Word Left */
466			fastcall u_int MTS_PROTO(sdl)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
467			{
468			m_uint64_t d_mask,naddr;
469			m_uint64_t data;
470			u_int r_shift;
471			void *haddr;
472			u_int exc;
473
474			naddr = vaddr & ~(0x07);
475			haddr = MTS_PROTO(access)(cpu,naddr,MIPS_MEMOP_SDL,8,MTS_READ,&data,&exc);
476			if (unlikely(exc)) return(exc);
477
478			if (likely(haddr != NULL))
479			data = vmtoh64((m_uint64_t )haddr);
480
481			r_shift = (vaddr & 0x07) << 3;
482			d_mask = 0xffffffffffffffffULL >> r_shift;
483
484			data &= ~d_mask;
485			data \|= cpu->gpr[reg] >> r_shift;
486
487			haddr = MTS_PROTO(access)(cpu,naddr,MIPS_MEMOP_SDL,8,MTS_WRITE,&data,&exc);
488			if (likely(haddr != NULL)) (m_uint64_t )haddr = htovm64(data);
489			return(exc);
490			}
491
492			/* SDR: Store Double-Word Right */
493			fastcall u_int MTS_PROTO(sdr)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
494			{
495			m_uint64_t d_mask,naddr;
496			m_uint64_t data;
497			u_int r_shift;
498			void *haddr;
499			u_int exc;
500
501			naddr = vaddr & ~(0x07);
502			haddr = MTS_PROTO(access)(cpu,naddr,MIPS_MEMOP_SDR,8,MTS_READ,&data,&exc);
503			if (unlikely(exc)) return(exc);
504
505			if (likely(haddr != NULL))
506			data = vmtoh64((m_uint64_t )haddr);
507
508			r_shift = ((vaddr & 0x07) + 1) << 3;
509			d_mask = 0xffffffffffffffffULL >> r_shift;
510
511			data &= d_mask;
512			data \|= cpu->gpr[reg] << (64 - r_shift);
513
514			haddr = MTS_PROTO(access)(cpu,naddr,MIPS_MEMOP_SDR,8,MTS_WRITE,&data,&exc);
515			if (likely(haddr != NULL)) (m_uint64_t )haddr = htovm64(data);
516			return(exc);
517			}
518
519			/* LL: Load Linked */
520			fastcall u_int MTS_PROTO(ll)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
521			{
522			m_uint64_t data;
523			void *haddr;
524			u_int exc;
525
526			haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_LL,4,MTS_READ,&data,&exc);
527			if (likely(haddr != NULL)) data = vmtoh32((m_uint32_t )haddr);
528
529			if (likely(!exc)) {
530			cpu->gpr[reg] = sign_extend(data,32);
531			cpu->ll_bit = 1;
532			}
533
534			return(exc);
535			}
536
537			/* SC: Store Conditional */
538			fastcall u_int MTS_PROTO(sc)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
539			{
540			m_uint64_t data;
541			void *haddr;
542			u_int exc = 0;
543
544			if (cpu->ll_bit) {
545			data = cpu->gpr[reg] & 0xffffffff;
546			haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_SC,4,MTS_WRITE,
547			&data,&exc);
548			if (likely(haddr != NULL)) (m_uint32_t )haddr = htovm32(data);
549			}
550
551			if (likely(!exc))
552			cpu->gpr[reg] = cpu->ll_bit;
553			return(exc);
554			}
555
556			/* SDC1: Store Double-Word from Coprocessor 1 */
557			fastcall u_int MTS_PROTO(sdc1)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
558			{
559			m_uint64_t data;
560			void *haddr;
561			u_int exc;
562
563			data = cpu->fpu.reg[reg];
564			haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_SDC1,8,MTS_WRITE,
565			&data,&exc);
566			if (likely(haddr != NULL)) (m_uint64_t )haddr = htovm64(data);
567			return(exc);
568			}
569
570			/* CACHE: Cache operation */
571			fastcall u_int MTS_PROTO(cache)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int op)
572			{
573	dpavlin	7	mips64_jit_tcb_t *block;
574	dpavlin	4	m_uint32_t phys_page;
575
576			#if DEBUG_CACHE
577	dpavlin	7	cpu_log(cpu->gen,
578			"MTS","CACHE: PC=0x%llx, vaddr=0x%llx, cache=%u, code=%u\n",
579	dpavlin	4	cpu->pc, vaddr, op & 0x3, op >> 2);
580			#endif
581
582			if (!cpu->translate(cpu,vaddr,&phys_page)) {
583			if ((phys_page < 1048576) && cpu->exec_phys_map) {
584			block = cpu->exec_phys_map[phys_page];
585
586			if (block) {
587			if ((cpu->pc < block->start_pc) \|\|
588			((cpu->pc - block->start_pc) >= MIPS_MIN_PAGE_SIZE))
589			{
590			#if DEBUG_CACHE
591	dpavlin	7	cpu_log(cpu->gen,"MTS",
592	dpavlin	4	"CACHE: removing compiled page at 0x%llx, pc=0x%llx\n",
593			block->start_pc,cpu->pc);
594			#endif
595			cpu->exec_phys_map[phys_page] = NULL;
596	dpavlin	7	mips64_jit_tcb_free(cpu,block,TRUE);
597	dpavlin	4	}
598			else
599			{
600			#if DEBUG_CACHE
601	dpavlin	7	cpu_log(cpu->gen,"MTS",
602	dpavlin	4	"CACHE: trying to remove page 0x%llx with pc=0x%llx\n",
603			block->start_pc,cpu->pc);
604			#endif
605			}
606			}
607			}
608			}
609
610			return(0);
611			}
612
613			/* === MTS Cache Management ============================================= */
614
615			/* MTS map/unmap/rebuild "API" functions */
616			void MTS_PROTO(api_map)(cpu_mips_t *cpu,m_uint64_t vaddr,m_uint64_t paddr,
617			m_uint32_t len,int cache_access,int tlb_index)
618			{
619			/* nothing to do, the cache will be filled on-the-fly */
620			}
621
622			void MTS_PROTO(api_unmap)(cpu_mips_t *cpu,m_uint64_t vaddr,m_uint32_t len,
623			m_uint32_t val,int tlb_index)
624			{
625			/* Invalidate the TLB entry or the full cache if no index is specified */
626			if (tlb_index != -1)
627			MTS_PROTO(invalidate_tlb_entry)(cpu,tlb_index);
628			else
629			MTS_PROTO(invalidate_cache)(cpu);
630			}
631
632	dpavlin	7	void MTS_PROTO(api_rebuild)(cpu_gen_t *cpu)
633	dpavlin	4	{
634	dpavlin	7	MTS_PROTO(invalidate_cache)(CPU_MIPS64(cpu));
635	dpavlin	4	}
636
637			/* ======================================================================== */
638
639			/* Initialize memory access vectors */
640			void MTS_PROTO(init_memop_vectors)(cpu_mips_t *cpu)
641			{
642			/* XXX TODO:
643			* - LD/SD forbidden in Supervisor/User modes with 32-bit addresses.
644			*/
645
646			cpu->addr_mode = MTS_ADDR_SIZE;
647
648			/* API vectors */
649			cpu->mts_map = MTS_PROTO(api_map);
650			cpu->mts_unmap = MTS_PROTO(api_unmap);
651
652	dpavlin	7	/* Memory lookup operation */
653	dpavlin	4	cpu->mem_op_lookup = MTS_PROTO(lookup);
654
655			/* Translation operation */
656			cpu->translate = MTS_PROTO(translate);
657
658			/* Shutdown operation */
659			cpu->mts_shutdown = MTS_PROTO(shutdown);
660
661	dpavlin	7	/* Rebuild MTS data structures */
662			cpu->gen->mts_rebuild = MTS_PROTO(api_rebuild);
663
664	dpavlin	4	/* Show statistics */
665	dpavlin	7	cpu->gen->mts_show_stats = MTS_PROTO(show_stats);
666	dpavlin	4
667			/* Load Operations */
668			cpu->mem_op_fn[MIPS_MEMOP_LB] = MTS_PROTO(lb);
669			cpu->mem_op_fn[MIPS_MEMOP_LBU] = MTS_PROTO(lbu);
670			cpu->mem_op_fn[MIPS_MEMOP_LH] = MTS_PROTO(lh);
671			cpu->mem_op_fn[MIPS_MEMOP_LHU] = MTS_PROTO(lhu);
672			cpu->mem_op_fn[MIPS_MEMOP_LW] = MTS_PROTO(lw);
673			cpu->mem_op_fn[MIPS_MEMOP_LWU] = MTS_PROTO(lwu);
674			cpu->mem_op_fn[MIPS_MEMOP_LD] = MTS_PROTO(ld);
675			cpu->mem_op_fn[MIPS_MEMOP_LDL] = MTS_PROTO(ldl);
676			cpu->mem_op_fn[MIPS_MEMOP_LDR] = MTS_PROTO(ldr);
677
678			/* Store Operations */
679			cpu->mem_op_fn[MIPS_MEMOP_SB] = MTS_PROTO(sb);
680			cpu->mem_op_fn[MIPS_MEMOP_SH] = MTS_PROTO(sh);
681			cpu->mem_op_fn[MIPS_MEMOP_SW] = MTS_PROTO(sw);
682			cpu->mem_op_fn[MIPS_MEMOP_SD] = MTS_PROTO(sd);
683
684			/* Load Left/Right operations */
685			cpu->mem_op_fn[MIPS_MEMOP_LWL] = MTS_PROTO(lwl);
686			cpu->mem_op_fn[MIPS_MEMOP_LWR] = MTS_PROTO(lwr);
687			cpu->mem_op_fn[MIPS_MEMOP_LDL] = MTS_PROTO(ldl);
688			cpu->mem_op_fn[MIPS_MEMOP_LDR] = MTS_PROTO(ldr);
689
690			/* Store Left/Right operations */
691			cpu->mem_op_fn[MIPS_MEMOP_SWL] = MTS_PROTO(swl);
692			cpu->mem_op_fn[MIPS_MEMOP_SWR] = MTS_PROTO(swr);
693			cpu->mem_op_fn[MIPS_MEMOP_SDL] = MTS_PROTO(sdl);
694			cpu->mem_op_fn[MIPS_MEMOP_SDR] = MTS_PROTO(sdr);
695
696			/* LL/SC - Load Linked / Store Conditional */
697			cpu->mem_op_fn[MIPS_MEMOP_LL] = MTS_PROTO(ll);
698			cpu->mem_op_fn[MIPS_MEMOP_SC] = MTS_PROTO(sc);
699
700			/* Coprocessor 1 memory access functions */
701			cpu->mem_op_fn[MIPS_MEMOP_LDC1] = MTS_PROTO(ldc1);
702			cpu->mem_op_fn[MIPS_MEMOP_SDC1] = MTS_PROTO(sdc1);
703
704			/* Cache Operation */
705			cpu->mem_op_fn[MIPS_MEMOP_CACHE] = MTS_PROTO(cache);
706			}
707
708			#undef MTS_ADDR_SIZE
709	dpavlin	7	#undef MTS_NAME
710			#undef MTS_NAME_UP
711	dpavlin	4	#undef MTS_PROTO
712			#undef MTS_PROTO_UP
713			#undef MTS_ENTRY
714			#undef MTS_CHUNK