/[dynamips]/upstream/dynamips-0.2.7-RC2/memory.c
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Annotation of /upstream/dynamips-0.2.7-RC2/memory.c

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1 - (hide annotations)
Sat Oct 6 16:01:44 2007 UTC (16 years, 5 months ago) by dpavlin
Original Path: upstream/dynamips-0.2.5/memory.c
File MIME type: text/plain
File size: 69903 byte(s)
import 0.2.5 from upstream

1 dpavlin 1 /*
2     * Cisco 7200 (Predator) simulation platform.
3     * Copyright (c) 2005,2006 Christophe Fillot (cf@utc.fr)
4     */
5    
6     #define _GNU_SOURCE
7     #include <stdio.h>
8     #include <stdlib.h>
9     #include <unistd.h>
10     #include <string.h>
11     #include <sys/types.h>
12     #include <sys/stat.h>
13     #include <sys/mman.h>
14     #include <fcntl.h>
15     #include <assert.h>
16    
17     #include "mips64.h"
18     #include "dynamips.h"
19     #include "memory.h"
20     #include "device.h"
21     #include "cpu.h"
22     #include "cp0.h"
23     #include "vm.h"
24    
25     /* Record a memory access */
26     void memlog_rec_access(cpu_mips_t *cpu,m_uint64_t vaddr,m_uint64_t data,
27     m_uint32_t op_size,m_uint32_t op_type)
28     {
29     memlog_access_t *acc;
30    
31     acc = &cpu->memlog_array[cpu->memlog_pos];
32     acc->pc = cpu->pc;
33     acc->vaddr = vaddr;
34     acc->data = data;
35     acc->op_size = op_size;
36     acc->op_type = op_type;
37     acc->data_valid = (op_type == MTS_WRITE);
38    
39     cpu->memlog_pos = (cpu->memlog_pos + 1) & (MEMLOG_COUNT - 1);
40     }
41    
42     /* Show the latest memory accesses */
43     void memlog_dump(cpu_mips_t *cpu)
44     {
45     memlog_access_t *acc;
46     char s_data[64];
47     u_int i,pos;
48    
49     for(i=0;i<MEMLOG_COUNT;i++) {
50     pos = cpu->memlog_pos + i;
51     pos &= (MEMLOG_COUNT-1);
52     acc = &cpu->memlog_array[pos];
53    
54     if (cpu->pc) {
55     if (acc->data_valid)
56     snprintf(s_data,sizeof(s_data),"0x%llx",acc->data);
57     else
58     snprintf(s_data,sizeof(s_data),"XXXXXXXX");
59    
60     printf("CPU%u: pc=0x%8.8llx, vaddr=0x%8.8llx, "
61     "size=%u, type=%s, data=%s\n",
62     cpu->id,acc->pc,acc->vaddr,acc->op_size,
63     (acc->op_type == MTS_READ) ? "read " : "write",
64     s_data);
65     }
66     }
67     }
68    
69     /* Update the data obtained by a read access */
70     void memlog_update_read(cpu_mips_t *cpu,m_iptr_t raddr)
71     {
72     memlog_access_t *acc;
73    
74     acc = &cpu->memlog_array[(cpu->memlog_pos-1) & (MEMLOG_COUNT-1)];
75    
76     if (acc->op_type == MTS_READ)
77     {
78     switch(acc->op_size) {
79     case 1:
80     acc->data = *(m_uint8_t *)raddr;
81     break;
82     case 2:
83     acc->data = vmtoh16(*(m_uint16_t *)raddr);
84     break;
85     case 4:
86     acc->data = vmtoh32(*(m_uint32_t *)raddr);
87     break;
88     case 8:
89     acc->data = vmtoh64(*(m_uint64_t *)raddr);
90     break;
91     }
92    
93     acc->data_valid = TRUE;
94     }
95     }
96    
97     /* MTS access with special access mask */
98     void mts_access_special(cpu_mips_t *cpu,m_uint64_t vaddr,m_uint32_t mask,
99     u_int op_code,u_int op_type,u_int op_size,
100     m_uint64_t *data,u_int *exc)
101     {
102     switch(mask) {
103     case MTS_ACC_U:
104     #if DEBUG_MTS_ACC_U
105     if (op_type == MTS_READ)
106     cpu_log(cpu,"MTS","read access to undefined address 0x%llx at "
107     "pc=0x%llx (size=%u)\n",vaddr,cpu->pc,op_size);
108     else
109     cpu_log(cpu,"MTS","write access to undefined address 0x%llx at "
110     "pc=0x%llx, value=0x%8.8llx (size=%u)\n",
111     vaddr,cpu->pc,*data,op_size);
112     #endif
113     if (op_type == MTS_READ)
114     *data = 0;
115     break;
116    
117     case MTS_ACC_T:
118     if (op_code != MIPS_MEMOP_LOOKUP) {
119     #if DEBUG_MTS_ACC_T
120     cpu_log(cpu,"MTS","TLB exception for address 0x%llx at pc=0x%llx "
121     "(%s access, size=%u)\n",
122     vaddr,cpu->pc,(op_type == MTS_READ) ?
123     "read":"write",op_size);
124     mips64_dump_regs(cpu);
125     #if MEMLOG_ENABLE
126     memlog_dump(cpu);
127     #endif
128     #endif
129     cpu->cp0.reg[MIPS_CP0_BADVADDR] = vaddr;
130    
131     if (op_type == MTS_READ)
132     mips64_trigger_exception(cpu,MIPS_CP0_CAUSE_TLB_LOAD,0);
133     else
134     mips64_trigger_exception(cpu,MIPS_CP0_CAUSE_TLB_SAVE,0);
135     }
136    
137     *exc = 1;
138     break;
139    
140     case MTS_ACC_AE:
141     if (op_code != MIPS_MEMOP_LOOKUP) {
142     #if DEBUG_MTS_ACC_AE
143     cpu_log(cpu,"MTS","AE exception for address 0x%llx at pc=0x%llx "
144     "(%s access)\n",
145     vaddr,cpu->pc,(op_type == MTS_READ) ? "read":"write");
146     #endif
147     cpu->cp0.reg[MIPS_CP0_BADVADDR] = vaddr;
148    
149     if (op_type == MTS_READ)
150     mips64_trigger_exception(cpu,MIPS_CP0_CAUSE_ADDR_LOAD,0);
151     else
152     mips64_trigger_exception(cpu,MIPS_CP0_CAUSE_ADDR_SAVE,0);
153     }
154    
155     *exc = 1;
156     break;
157     }
158     }
159    
160     /* Allocate an L1 array */
161     mts32_l1_array_t *mts32_alloc_l1_array(m_iptr_t val)
162     {
163     mts32_l1_array_t *p;
164     u_int i;
165    
166     if (!(p = malloc(sizeof(mts32_l1_array_t))))
167     return NULL;
168    
169     for(i=0;i<(1 << MTS32_LEVEL1_BITS);i++)
170     p->entry[i] = val;
171    
172     return p;
173     }
174    
175     /* Allocate an L2 array */
176     mts32_l2_array_t *mts32_alloc_l2_array(cpu_mips_t *cpu,m_iptr_t val)
177     {
178     mts32_l2_array_t *p;
179     u_int i;
180    
181     if (cpu->mts32_l2_free_list) {
182     p = cpu->mts32_l2_free_list;
183     cpu->mts32_l2_free_list = p->next;
184     } else {
185     if (!(p = m_memalign((1 << MTS_FLAG_BITS),sizeof(*p))))
186     return NULL;
187     }
188    
189     for(i=0;i<(1 << MTS32_LEVEL2_BITS);i++)
190     p->entry[i] = val;
191    
192     return p;
193     }
194    
195     /* Free an L1 array */
196     void mts32_free_l1_array(mts32_l1_array_t *array)
197     {
198     u_int i;
199    
200     if (array != NULL) {
201     for(i=0;i<(1<<MTS32_LEVEL1_BITS);i++)
202     if (array->entry[i] & MTS_CHAIN_MASK)
203     free((void *)(array->entry[i] & ~MTS_CHAIN_MASK));
204    
205     free(array);
206     }
207     }
208    
209     /* Free an L2 array */
210     void mts32_free_l2_array(cpu_mips_t *cpu,mts32_l2_array_t *array)
211     {
212     array->next = cpu->mts32_l2_free_list;
213     cpu->mts32_l2_free_list = array;
214     }
215    
216     /* Set an L1 entry */
217     static void mts32_set_l1_data(cpu_mips_t *cpu,m_uint32_t start,m_uint32_t len,
218     m_iptr_t val)
219     {
220     mts32_l1_array_t *p = cpu->mts_l1_ptr;
221     m_uint32_t pos;
222     m_iptr_t p2;
223    
224     while(len > 0) {
225     pos = start >> (MTS32_LEVEL2_BITS + MTS32_OFFSET_BITS);
226    
227     if (pos >= (1 << MTS32_LEVEL1_BITS))
228     break;
229    
230     /* free a possible L2 array */
231     if (p->entry[pos] & MTS_CHAIN_MASK) {
232     p2 = p->entry[pos] & ~MTS_CHAIN_MASK;
233     mts32_free_l2_array(cpu,(mts32_l2_array_t *)p2);
234     }
235    
236     p->entry[pos] = val;
237     start += MTS32_LEVEL1_SIZE;
238     len -= MTS32_LEVEL1_SIZE;
239     }
240     }
241    
242     /* Fork an L1 array */
243     static int mts32_fork_l1_array(cpu_mips_t *cpu,u_int l1_pos)
244     {
245     mts32_l1_array_t *p1;
246     mts32_l2_array_t *p2;
247     m_iptr_t entry,val;
248     u_int i;
249    
250     p1 = cpu->mts_l1_ptr;
251     entry = p1->entry[l1_pos];
252     val = ((entry & MTS_ACC_MASK) != MTS_ACC_OK) ? entry : 0;
253    
254     if (!(p2 = mts32_alloc_l2_array(cpu,val)))
255     return(-1);
256    
257     /* mts32_alloc_l2_array() did the job for us */
258     if (!val) {
259     for(i=0;i<(1 << MTS32_LEVEL2_BITS);i++)
260     p2->entry[i] = entry + (1 << MTS32_OFFSET_BITS);
261     }
262    
263     p1->entry[l1_pos] = (m_iptr_t)p2 | MTS_CHAIN_MASK;
264     return(0);
265     }
266    
267     /* Set address error on a complete level 1 array */
268     void mts32_set_l1_ae(cpu_mips_t *cpu)
269     {
270     mts32_l1_array_t *p1 = cpu->mts_l1_ptr;
271     u_int i;
272    
273     for(i=0;i<(1<<MTS32_LEVEL1_BITS);i++)
274     p1->entry[i] = MTS_ACC_AE;
275     }
276    
277     /* Set an L2 entry */
278     static int mts32_set_l2_entry(cpu_mips_t *cpu,m_uint64_t vaddr,m_iptr_t val)
279     {
280     m_uint32_t naddr = vaddr & 0xffffffff;
281     m_uint32_t l1_pos,l2_pos;
282     mts32_l1_array_t *p1;
283     mts32_l2_array_t *p2;
284     m_iptr_t entry;
285    
286     p1 = cpu->mts_l1_ptr;
287    
288     l1_pos = naddr >> (MTS32_LEVEL2_BITS + MTS32_OFFSET_BITS);
289     l2_pos = (naddr >> MTS32_OFFSET_BITS) & ((1 << MTS32_LEVEL2_BITS) - 1);
290    
291     entry = p1->entry[l1_pos];
292    
293     if (!(entry & MTS_CHAIN_MASK)) {
294     if (mts32_fork_l1_array(cpu,l1_pos) == -1) {
295     fprintf(stderr,"mts32_set_l2_entry: unable to fork L1 entry.\n");
296     return(-1);
297     }
298    
299     entry = p1->entry[l1_pos];
300     }
301    
302     p2 = (mts32_l2_array_t *)(entry & MTS_ADDR_MASK);
303     p2->entry[l2_pos] = val;
304     return(0);
305     }
306    
307     /* Initialize an empty MTS32 subsystem */
308     int mts32_init_empty(cpu_mips_t *cpu)
309     {
310     if (cpu->state == MIPS_CPU_RUNNING) {
311     cpu_log(cpu,"MTS","trying to reset MTS while the CPU is online.\n");
312     return(-1);
313     }
314    
315     mts32_free_l1_array(cpu->mts_l1_ptr);
316    
317     /* Allocate a new L1 array */
318     cpu->mts_l1_ptr = mts32_alloc_l1_array(0);
319    
320     if (!cpu->mts_l1_ptr)
321     return(-1);
322    
323     /* Address Error on complete address space for now */
324     mts32_set_l1_ae(cpu);
325     return(0);
326     }
327    
328     /* Free memory used by MTS32 */
329     void mts32_shutdown(cpu_mips_t *cpu)
330     {
331     mts32_l2_array_t *array,*next;
332    
333     /* Free L1/L2 entries */
334     if (cpu->mts_l1_ptr) {
335     mts32_free_l1_array(cpu->mts_l1_ptr);
336     cpu->mts_l1_ptr = NULL;
337     }
338    
339     /* Free arrays that are sitting in the free list */
340     for(array=cpu->mts32_l2_free_list;array;array=next) {
341     next = array->next;
342     free(array);
343     }
344    
345     cpu->mts32_l2_free_list = NULL;
346     }
347    
348     /* Map a device at the specified virtual address */
349     void mts32_map_device(cpu_mips_t *cpu,u_int dev_id,m_uint64_t vaddr,
350     m_uint32_t offset,m_uint32_t len)
351     {
352     struct vdevice *dev;
353     m_iptr_t val;
354    
355     if (!(dev = dev_get_by_id(cpu->vm,dev_id)) || !dev->phys_len)
356     return;
357    
358     #if DEBUG_MTS_MAP_DEV
359     cpu_log(cpu,"MTS32",
360     "mapping device %s (offset=0x%x,len=0x%x) at vaddr 0x%llx\n",
361     dev->name,offset,len,vaddr);
362     #endif
363    
364     if (!dev->host_addr || (dev->flags & VDEVICE_FLAG_NO_MTS_MMAP))
365     val = (dev_id << MTS_DEVID_SHIFT) | MTS_DEV_MASK | MTS_ACC_OK;
366     else
367     val = dev->host_addr | MTS_ACC_OK;
368    
369     val += offset;
370    
371     while(len > 0) {
372     if (!(vaddr & MTS32_LEVEL1_MASK) && !(len & MTS32_LEVEL1_MASK)) {
373     mts32_set_l1_data(cpu,vaddr,MTS32_LEVEL1_SIZE,val);
374     vaddr += MTS32_LEVEL1_SIZE;
375     val += MTS32_LEVEL1_SIZE;
376     len -= MTS32_LEVEL1_SIZE;
377     } else {
378     mts32_set_l2_entry(cpu,vaddr,val);
379     vaddr += MTS32_LEVEL2_SIZE;
380     val += MTS32_LEVEL2_SIZE;
381     len -= MTS32_LEVEL2_SIZE;
382     }
383     }
384     }
385    
386     /* Map a physical address to the specified virtual address */
387     void mts32_map(cpu_mips_t *cpu,m_uint64_t vaddr,
388     m_uint64_t paddr,m_uint32_t len,
389     int cache_access)
390     {
391     struct vdevice *dev,*next_dev;
392     m_uint32_t dev_offset,clen;
393    
394     while(len > 0)
395     {
396     #if DEBUG_MTS_MAP_VIRT
397     cpu_log(cpu,"MTS32",
398     "mts32_map: vaddr=0x%llx, paddr=0x%llx, len=0x%x, cache=%d\n",
399     vaddr,paddr,len,cache_access);
400     #endif
401     dev = dev_lookup(cpu->vm,paddr,cache_access);
402     next_dev = dev_lookup_next(cpu->vm,paddr,dev,cache_access);
403    
404     if (next_dev)
405     clen = m_min(len,next_dev->phys_addr-paddr);
406     else
407     clen = len;
408    
409     if (!dev) {
410     mts32_unmap(cpu,vaddr,clen,MTS_ACC_U);
411     } else {
412     dev_offset = paddr - dev->phys_addr;
413     clen = m_min(clen,dev->phys_len);
414     mts32_map_device(cpu,dev->id,vaddr,dev_offset,clen);
415     }
416    
417     vaddr += clen;
418     paddr += clen;
419     len -= clen;
420     }
421     }
422    
423     /* Unmap a memory zone */
424     void mts32_unmap(cpu_mips_t *cpu,m_uint64_t vaddr,m_uint32_t len,
425     m_uint32_t val)
426     {
427     while(len > 0)
428     {
429     #if DEBUG_MTS_MAP_VIRT
430     cpu_log(cpu,"MTS32","mts32_unmap: vaddr=0x%llx, len=0x%x\n",vaddr,len);
431     #endif
432     if (!(vaddr & MTS32_LEVEL1_MASK) && !(len & MTS32_LEVEL1_MASK)) {
433     mts32_set_l1_data(cpu,vaddr,MTS32_LEVEL1_SIZE,val);
434     vaddr += MTS32_LEVEL1_SIZE;
435     len -= MTS32_LEVEL1_SIZE;
436     } else {
437     mts32_set_l2_entry(cpu,vaddr,val);
438     vaddr += MTS32_LEVEL2_SIZE;
439     len -= MTS32_LEVEL2_SIZE;
440     }
441     }
442     }
443    
444     /* Map all devices for kernel mode */
445     void mts32_km_map_all_dev(cpu_mips_t *cpu)
446     {
447     /* KSEG0: cached accesses */
448     mts32_map(cpu,MIPS_KSEG0_BASE,0,MTS_SIZE_512M,TRUE);
449    
450     /* KSEG1: uncached accesses */
451     mts32_map(cpu,MIPS_KSEG1_BASE,0,MTS_SIZE_512M,FALSE);
452     }
453    
454     /* MTS32 raw lookup */
455     static forced_inline
456     void *mts32_raw_lookup(cpu_mips_t *cpu,mts32_l1_array_t *p1,m_uint64_t vaddr)
457     {
458     m_uint32_t naddr = vaddr & 0xffffffff;
459     m_uint32_t l1_pos,l2_pos,shift;
460     mts32_l2_array_t *p2;
461     m_iptr_t entry,haddr;
462     m_uint64_t data;
463     u_int dev_id;
464    
465     shift = MTS32_LEVEL2_BITS + MTS32_OFFSET_BITS;
466     l1_pos = naddr >> shift;
467     entry = p1->entry[l1_pos];
468    
469     if (unlikely((entry & MTS_ACC_MASK) != MTS_ACC_OK))
470     return NULL;
471    
472     if (entry & MTS_CHAIN_MASK) {
473     p2 = (mts32_l2_array_t *)(entry & MTS_ADDR_MASK);
474     l2_pos = (naddr >> MTS32_OFFSET_BITS) & ((1 << MTS32_LEVEL2_BITS) - 1);
475     entry = p2->entry[l2_pos];
476     shift = MTS32_OFFSET_BITS;
477     }
478    
479     if (unlikely((entry & MTS_ACC_MASK) != MTS_ACC_OK))
480     return NULL;
481    
482     /* device access */
483     if (unlikely(entry & MTS_DEV_MASK)) {
484     dev_id = (entry & MTS_DEVID_MASK) >> MTS_DEVID_SHIFT;
485     haddr = (entry & MTS_DEVOFF_MASK);
486     haddr += (naddr & ((1 << shift) - 1));
487     return(dev_access_fast(cpu,dev_id,haddr,4,MTS_READ,&data));
488     }
489    
490     haddr = entry & MTS_ADDR_MASK;
491     haddr += (naddr & ((1 << shift) - 1));
492     return((void *)haddr);
493     }
494    
495     /* MTS32 access */
496     static forced_inline void *mts32_access(cpu_mips_t *cpu,m_uint64_t vaddr,
497     u_int op_code,u_int op_size,
498     u_int op_type,m_uint64_t *data,
499     u_int *exc)
500     {
501     m_uint32_t naddr = vaddr & 0xffffffff;
502     m_uint32_t l1_pos,l2_pos,mask,shift;
503     mts32_l1_array_t *p1 = cpu->mts_l1_ptr;
504     mts32_l2_array_t *p2;
505     m_iptr_t entry,haddr;
506     u_int dev_id;
507    
508     #if MEMLOG_ENABLE
509     /* Record the memory access */
510     memlog_rec_access(cpu,vaddr,*data,op_size,op_type);
511     #endif
512    
513     *exc = 0;
514     shift = MTS32_LEVEL2_BITS + MTS32_OFFSET_BITS;
515     l1_pos = naddr >> shift;
516     entry = p1->entry[l1_pos];
517    
518     if (unlikely((mask = (entry & MTS_ACC_MASK)) != MTS_ACC_OK)) {
519     mts_access_special(cpu,vaddr,mask,op_code,op_type,op_size,data,exc);
520     return NULL;
521     }
522    
523     /* do we have a level 2 entry ? */
524     if (entry & MTS_CHAIN_MASK) {
525     p2 = (mts32_l2_array_t *)(entry & MTS_ADDR_MASK);
526     l2_pos = (naddr >> MTS32_OFFSET_BITS) & ((1 << MTS32_LEVEL2_BITS) - 1);
527     entry = p2->entry[l2_pos];
528     shift = MTS32_OFFSET_BITS;
529    
530     if (unlikely((mask = (entry & MTS_ACC_MASK)) != MTS_ACC_OK)) {
531     mts_access_special(cpu,vaddr,mask,op_code,op_type,op_size,data,exc);
532     return NULL;
533     }
534     }
535    
536     /* device access */
537     if (unlikely(entry & MTS_DEV_MASK)) {
538     dev_id = (entry & MTS_DEVID_MASK) >> MTS_DEVID_SHIFT;
539     haddr = (entry & MTS_DEVOFF_MASK);
540     haddr += (naddr & ((1 << shift) - 1));
541    
542     #if DEBUG_MTS_DEV
543     cpu_log(cpu,"MTS32",
544     "device access: vaddr=0x%llx, pc=0x%llx, dev_offset=0x%x\n",
545     vaddr,cpu->pc,haddr);
546     #endif
547     return(dev_access_fast(cpu,dev_id,haddr,op_size,op_type,data));
548     }
549    
550     /* raw memory access */
551     haddr = entry & MTS_ADDR_MASK;
552     haddr += (naddr & ((1 << shift) - 1));
553     #if MEMLOG_ENABLE
554     memlog_update_read(cpu,haddr);
555     #endif
556     return((void *)haddr);
557     }
558    
559     /* Memory lookup */
560     void *mts32_lookup(cpu_mips_t *cpu,m_uint64_t vaddr)
561     {
562     return(mts32_raw_lookup(cpu,cpu->mts_l1_ptr,vaddr));
563     }
564    
565     /* Initialize the MTS64 subsystem for the specified CPU */
566     int mts64_init(cpu_mips_t *cpu)
567     {
568     size_t len;
569    
570     /* Initialize the cache entries to 0 (empty) */
571     len = MTS64_HASH_SIZE * sizeof(mts64_entry_t *);
572     if (!(cpu->mts64_cache = malloc(len)))
573     return(-1);
574    
575     memset(cpu->mts64_cache,0,len);
576     cpu->mts64_lookups = 0;
577     cpu->mts64_misses = 0;
578    
579     /* Reset the TLB reverse map (used for selective invalidations) */
580     memset(cpu->mts64_rmap,0,(cpu->cp0.tlb_entries * sizeof(mts64_entry_t *)));
581     return(0);
582     }
583    
584     /* Free memory used by MTS64 */
585     void mts64_shutdown(cpu_mips_t *cpu)
586     {
587     mts64_chunk_t *chunk,*next;
588     int i;
589    
590     /* Reset the reverse map */
591     for(i=0;i<cpu->cp0.tlb_entries;i++)
592     cpu->mts64_rmap[i] = NULL;
593    
594     /* Free the cache itself */
595     free(cpu->mts64_cache);
596     cpu->mts64_cache = NULL;
597    
598     /* Free the chunks */
599     for(chunk=cpu->mts64_chunk_list;chunk;chunk=next) {
600     next = chunk->next;
601     free(chunk);
602     }
603    
604     for(chunk=cpu->mts64_chunk_free_list;chunk;chunk=next) {
605     next = chunk->next;
606     free(chunk);
607     }
608    
609     cpu->mts64_chunk_list = cpu->mts64_chunk_free_list = NULL;
610     cpu->mts64_entry_free_list = NULL;
611     }
612    
613     /* Show MTS64 detailed information (debugging only!) */
614     void mts64_show_stats(cpu_mips_t *cpu)
615     {
616     mts64_chunk_t *chunk;
617     #if DEBUG_MTS_MAP_VIRT
618     mts64_entry_t *entry;
619     u_int i;
620     #endif
621     u_int count;
622    
623     printf("\nCPU%u: MTS64 statistics:\n",cpu->id);
624    
625     printf(" Total lookups: %llu, misses: %llu, efficiency: %g%%\n",
626     cpu->mts64_lookups, cpu->mts64_misses,
627     100 - ((double)(cpu->mts64_misses*100)/
628     (double)cpu->mts64_lookups));
629    
630     #if DEBUG_MTS_MAP_VIRT
631     /* Valid hash entries */
632     for(count=0,i=0;i<MTS64_HASH_SIZE;i++) {
633     if ((entry = cpu->mts64_cache[i]) != NULL) {
634     printf(" %4u: entry=%p, start=0x%16.16llx, "
635     "len=0x%8.8x (%6u bytes), action=0x%llx\n",
636     i,entry,entry->start,entry->mask,entry->mask+1,
637     (m_uint64_t)entry->action);
638     count++;
639     }
640     }
641    
642     printf(" %u/%u valid hash entries.\n",count,MTS64_HASH_SIZE);
643     #endif
644    
645     /* Number of chunks */
646     for(count=0,chunk=cpu->mts64_chunk_list;chunk;chunk=chunk->next)
647     count++;
648    
649     printf(" Number of chunks: %u\n",count);
650    
651     #if DEBUG_MTS_MAP_VIRT
652     /* Reverse map */
653     for(i=0;i<MIPS64_TLB_ENTRIES;i++) {
654     for(count=0,entry=cpu->mts64_rmap[i];entry;entry=entry->next)
655     count++;
656    
657     if (count > 0)
658     printf(" tlb_rmap[%u]: %u entries\n",i,count);
659     }
660     #endif
661     }
662    
663     /* Allocate a new chunk */
664     static int mts64_alloc_chunk(cpu_mips_t *cpu)
665     {
666     mts64_chunk_t *chunk;
667    
668     /* Try the free list first, then use standard allocation procedure */
669     if ((chunk = cpu->mts64_chunk_free_list) != NULL) {
670     cpu->mts64_chunk_free_list = chunk->next;
671     } else {
672     if (!(chunk = malloc(sizeof(*chunk))))
673     return(-1);
674     }
675    
676     chunk->count = 0;
677     chunk->next = cpu->mts64_chunk_list;
678     cpu->mts64_chunk_list = chunk;
679     return(0);
680     }
681    
682     /* Allocate a new entry */
683     static mts64_entry_t *mts64_alloc_entry(cpu_mips_t *cpu)
684     {
685     mts64_chunk_t *chunk = cpu->mts64_chunk_list;
686     mts64_entry_t *entry;
687    
688     /* First, try to allocate the entry from the free list */
689     if ((entry = cpu->mts64_entry_free_list) != NULL) {
690     cpu->mts64_entry_free_list = cpu->mts64_entry_free_list->next;
691     return entry;
692     }
693    
694     /* A new chunk is required */
695     if (!chunk || (chunk->count == MTS64_CHUNK_SIZE)) {
696     if (mts64_alloc_chunk(cpu) == -1)
697     return NULL;
698    
699     chunk = cpu->mts64_chunk_list;
700     }
701    
702     entry = &chunk->entry[chunk->count];
703     chunk->count++;
704     return entry;
705     }
706    
707     /* Invalidate the complete MTS64 cache */
708     void mts64_invalidate_cache(cpu_mips_t *cpu)
709     {
710     mts64_chunk_t *chunk;
711     size_t len;
712     u_int i;
713    
714     len = MTS64_HASH_SIZE * sizeof(mts64_entry_t *);
715     memset(cpu->mts64_cache,0,len);
716    
717     /* Move all chunks to the free list */
718     while((chunk = cpu->mts64_chunk_list) != NULL) {
719     cpu->mts64_chunk_list = chunk->next;
720     chunk->next = cpu->mts64_chunk_free_list;
721     cpu->mts64_chunk_free_list = chunk;
722     }
723    
724     /* Reset the free list of entries (since they are located in chunks) */
725     cpu->mts64_entry_free_list = NULL;
726    
727     /* Reset the reverse map */
728     for(i=0;i<cpu->cp0.tlb_entries;i++)
729     cpu->mts64_rmap[i] = NULL;
730     }
731    
732     /* Invalidate partially the MTS64 cache, given a TLB entry index */
733     void mts64_invalidate_tlb_entry(cpu_mips_t *cpu,u_int tlb_index)
734     {
735     mts64_entry_t *entry;
736    
737     for(entry=cpu->mts64_rmap[tlb_index];entry;entry=entry->next) {
738     *(entry->pself) = NULL;
739     if (!entry->next) {
740     entry->next = cpu->mts64_entry_free_list;
741     break;
742     }
743     }
744    
745     cpu->mts64_entry_free_list = cpu->mts64_rmap[tlb_index];
746     cpu->mts64_rmap[tlb_index] = NULL;
747     }
748    
749     /*
750     * MTS64 mapping.
751     *
752     * It is NOT inlined since it triggers a GCC bug on my config (x86, GCC 3.3.5)
753     */
754     static no_inline int mts64_map(cpu_mips_t *cpu,m_uint64_t vaddr,mts_map_t *map,
755     mts64_entry_t *entry)
756     {
757     struct vdevice *dev;
758     m_uint64_t lk_addr;
759     m_uint32_t poffset;
760    
761     lk_addr = map->paddr + (vaddr - map->vaddr);
762    
763     if (!(dev = dev_lookup(cpu->vm,lk_addr,map->cached)))
764     return(FALSE);
765    
766     if (map->paddr > dev->phys_addr) {
767     poffset = map->paddr - dev->phys_addr;
768     entry->start = map->vaddr;
769     entry->phys_page = map->paddr >> MIPS_MIN_PAGE_SHIFT;
770     entry->mask = ~((m_min(map->len,dev->phys_len - poffset)) - 1);
771     entry->action = poffset;
772     } else {
773     poffset = dev->phys_addr - map->paddr;
774     entry->start = map->vaddr + poffset;
775     entry->phys_page = (map->paddr + poffset) >> MIPS_MIN_PAGE_SHIFT;
776     entry->mask = ~((m_min(map->len - poffset,dev->phys_len)) - 1);
777     entry->action = 0;
778     }
779    
780     if (!dev->host_addr || (dev->flags & VDEVICE_FLAG_NO_MTS_MMAP))
781     entry->action += (dev->id << MTS_DEVID_SHIFT) | MTS_DEV_MASK;
782     else
783     entry->action += dev->host_addr;
784    
785     return(TRUE);
786     }
787    
788     /* MTS64 slow lookup */
789     static forced_inline
790     mts64_entry_t *mts64_slow_lookup(cpu_mips_t *cpu,m_uint64_t vaddr,
791     u_int op_code,u_int op_size,
792     u_int op_type,m_uint64_t *data,
793     u_int *exc)
794     {
795     m_uint32_t hash_bucket,zone,sub_zone,cca;
796     mts64_entry_t *entry,new_entry;
797     mts_map_t map;
798    
799     map.tlb_index = -1;
800     hash_bucket = MTS64_HASH(vaddr);
801     entry = cpu->mts64_cache[hash_bucket];
802     zone = vaddr >> 40;
803    
804     #if DEBUG_MTS_STATS
805     cpu->mts64_misses++;
806     #endif
807    
808     switch(zone) {
809     case 0x000000: /* xkuseg */
810     case 0x400000: /* xksseg */
811     case 0xc00000: /* xkseg */
812     /* trigger TLB exception if no matching entry found */
813     if (!cp0_tlb_lookup(cpu,vaddr,&map))
814     goto err_tlb;
815    
816     if (!mts64_map(cpu,vaddr,&map,&new_entry))
817     goto err_undef;
818     break;
819    
820     case 0xffffff:
821     sub_zone = (vaddr >> 29) & 0x7FF;
822    
823     switch(sub_zone) {
824     case 0x7fc: /* ckseg0 */
825     map.vaddr = sign_extend(MIPS_KSEG0_BASE,32);
826     map.paddr = 0;
827     map.len = MIPS_KSEG0_SIZE;
828     map.cached = TRUE;
829     if (!mts64_map(cpu,vaddr,&map,&new_entry))
830     goto err_undef;
831     break;
832    
833     case 0x7fd: /* ckseg1 */
834     map.vaddr = sign_extend(MIPS_KSEG1_BASE,32);
835     map.paddr = 0;
836     map.len = MIPS_KSEG1_SIZE;
837     map.cached = FALSE;
838     if (!mts64_map(cpu,vaddr,&map,&new_entry))
839     goto err_undef;
840     break;
841    
842     case 0x7fe: /* cksseg */
843     case 0x7ff: /* ckseg3 */
844     /* trigger TLB exception if no matching entry found */
845     if (!cp0_tlb_lookup(cpu,vaddr,&map))
846     goto err_tlb;
847    
848     if (!mts64_map(cpu,vaddr,&map,&new_entry))
849     goto err_undef;
850     break;
851    
852     default:
853     /* Invalid zone: generate Address Error (AE) exception */
854     goto err_address;
855     }
856     break;
857    
858     /* xkphys */
859     case 0x800000:
860     case 0x880000:
861     case 0x900000:
862     case 0x980000:
863     case 0xa00000:
864     case 0xa80000:
865     case 0xb00000:
866     case 0xb80000:
867     cca = (vaddr >> MIPS64_XKPHYS_CCA_SHIFT) & 0x03;
868     map.cached = mips64_cca_cached(cca);
869     map.vaddr = vaddr & MIPS64_XKPHYS_ZONE_MASK;
870     map.paddr = 0;
871     map.len = MIPS64_XKPHYS_PHYS_SIZE;
872     if (!mts64_map(cpu,vaddr,&map,&new_entry))
873     goto err_undef;
874     break;
875    
876     default:
877     /* Invalid zone: generate Address Error (AE) exception */
878     goto err_address;
879     }
880    
881     /* Get a new entry if necessary */
882     if (!entry) {
883     entry = mts64_alloc_entry(cpu);
884     entry->pself = entry->pprev = NULL;
885     entry->next = NULL;
886    
887     /* Store the entry in hash table for future use */
888     cpu->mts64_cache[hash_bucket] = entry;
889     } else {
890     /* Remove the entry from the reverse map list */
891     if (entry->pprev) {
892     if (entry->next)
893     entry->next->pprev = entry->pprev;
894    
895     *(entry->pprev) = entry->next;
896     }
897     }
898    
899     /* Add this entry to the reverse map list */
900     if (map.tlb_index != -1) {
901     entry->pself = &cpu->mts64_cache[hash_bucket];
902     entry->next = cpu->mts64_rmap[map.tlb_index];
903     entry->pprev = &cpu->mts64_rmap[map.tlb_index];
904     if (entry->next)
905     entry->next->pprev = &entry->next;
906     cpu->mts64_rmap[map.tlb_index] = entry;
907     }
908    
909     /* Fill the new entry or replace the previous */
910     entry->phys_page = new_entry.phys_page;
911     entry->start = new_entry.start;
912     entry->mask = new_entry.mask;
913     entry->action = new_entry.action;
914     return entry;
915    
916     err_undef:
917     mts_access_special(cpu,vaddr,MTS_ACC_U,op_code,op_type,op_size,data,exc);
918     return NULL;
919     err_address:
920     mts_access_special(cpu,vaddr,MTS_ACC_AE,op_code,op_type,op_size,data,exc);
921     return NULL;
922     err_tlb:
923     mts_access_special(cpu,vaddr,MTS_ACC_T,op_code,op_type,op_size,data,exc);
924     return NULL;
925     }
926    
927     /* MTS64 access */
928     static forced_inline void *mts64_access(cpu_mips_t *cpu,m_uint64_t vaddr,
929     u_int op_code,u_int op_size,
930     u_int op_type,m_uint64_t *data,
931     u_int *exc)
932     {
933     m_uint32_t hash_bucket;
934     mts64_entry_t *entry;
935     m_iptr_t haddr;
936     u_int dev_id;
937    
938     #if MEMLOG_ENABLE
939     /* Record the memory access */
940     memlog_rec_access(cpu,vaddr,*data,op_size,op_type);
941     #endif
942    
943     *exc = 0;
944     hash_bucket = MTS64_HASH(vaddr);
945     entry = cpu->mts64_cache[hash_bucket];
946    
947     #if DEBUG_MTS_STATS
948     cpu->mts64_lookups++;
949     #endif
950    
951     /* Slow lookup if nothing found in cache */
952     if (unlikely((!entry) ||
953     unlikely((vaddr & sign_extend(entry->mask,32)) != entry->start)))
954     {
955     entry = mts64_slow_lookup(cpu,vaddr,op_code,op_size,op_type,data,exc);
956     if (!entry) return NULL;
957     }
958    
959     /* Device access */
960     if (unlikely(entry->action & MTS_DEV_MASK)) {
961     dev_id = (entry->action & MTS_DEVID_MASK) >> MTS_DEVID_SHIFT;
962     haddr = entry->action & MTS_DEVOFF_MASK;
963     haddr += vaddr - entry->start;
964    
965     #if DEBUG_MTS_DEV
966     cpu_log(cpu,"MTS64",
967     "device access: vaddr=0x%llx, pc=0x%llx, dev_offset=0x%x\n",
968     vaddr,cpu->pc,haddr);
969     #endif
970     return(dev_access_fast(cpu,dev_id,haddr,op_size,op_type,data));
971     }
972    
973     /* Raw memory access */
974     haddr = entry->action & MTS_ADDR_MASK;
975     haddr += vaddr - entry->start;
976     #if MEMLOG_ENABLE
977     memlog_update_read(cpu,haddr);
978     #endif
979     return((void *)haddr);
980     }
981    
982     /* MTS64 lookup */
983     static void *mts64_lookup(cpu_mips_t *cpu,m_uint64_t vaddr)
984     {
985     m_uint64_t data;
986     u_int exc;
987     return(mts64_access(cpu,vaddr,MIPS_MEMOP_LOOKUP,4,MTS_READ,&data,&exc));
988     }
989    
990     /* MTS64 virtual address to physical page translation */
991     static fastcall int mts64_translate(cpu_mips_t *cpu,m_uint64_t vaddr,
992     m_uint32_t *phys_page)
993     {
994     m_uint32_t hash_bucket,offset;
995     mts64_entry_t *entry;
996     m_uint64_t data = 0;
997     u_int exc = 0;
998    
999     hash_bucket = MTS64_HASH(vaddr);
1000     entry = cpu->mts64_cache[hash_bucket];
1001    
1002     /* Slow lookup if nothing found in cache */
1003     if (unlikely((!entry) ||
1004     unlikely((vaddr & sign_extend(entry->mask,32)) != entry->start)))
1005     {
1006     entry = mts64_slow_lookup(cpu,vaddr,MIPS_MEMOP_LOOKUP,4,MTS_READ,
1007     &data,&exc);
1008     if (!entry)
1009     return(-1);
1010     }
1011    
1012     offset = vaddr - entry->start;
1013     *phys_page = entry->phys_page + (offset >> MIPS_MIN_PAGE_SHIFT);
1014     return(0);
1015     }
1016    
1017     /* === MIPS Memory Operations ============================================= */
1018    
1019     /* Macro for MTS access (32 or 64 bit) */
1020     #define MTS_ACCESS(X) mts##X##_access
1021     #define MTS_MEMOP(op) MTS_##op(32) MTS_##op(64)
1022    
1023     /* LB: Load Byte */
1024     #define MTS_LB(X) \
1025     fastcall u_int mts##X##_lb(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg) \
1026     { \
1027     m_uint64_t data; \
1028     void *haddr; \
1029     u_int exc; \
1030     \
1031     haddr = MTS_ACCESS(X)(cpu,vaddr,MIPS_MEMOP_LB,1,MTS_READ,&data,&exc); \
1032     if (likely(haddr != NULL)) data = *(m_uint8_t *)haddr; \
1033     if (likely(!exc)) cpu->gpr[reg] = sign_extend(data,8); \
1034     return(exc); \
1035     } \
1036    
1037     /* LBU: Load Byte Unsigned */
1038     #define MTS_LBU(X) \
1039     fastcall u_int mts##X##_lbu(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg) \
1040     { \
1041     m_uint64_t data; \
1042     void *haddr; \
1043     u_int exc; \
1044     \
1045     haddr = MTS_ACCESS(X)(cpu,vaddr,MIPS_MEMOP_LBU,1,MTS_READ,&data,&exc); \
1046     if (likely(haddr != NULL)) data = *(m_uint8_t *)haddr; \
1047     if (likely(!exc)) cpu->gpr[reg] = data & 0xff; \
1048     return(exc); \
1049     }
1050    
1051     /* LH: Load Half-Word */
1052     #define MTS_LH(X) \
1053     fastcall u_int mts##X##_lh(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg) \
1054     { \
1055     m_uint64_t data; \
1056     void *haddr; \
1057     u_int exc; \
1058     \
1059     haddr = MTS_ACCESS(X)(cpu,vaddr,MIPS_MEMOP_LH,2,MTS_READ,&data,&exc); \
1060     if (likely(haddr != NULL)) data = vmtoh16(*(m_uint16_t *)haddr); \
1061     if (likely(!exc)) cpu->gpr[reg] = sign_extend(data,16); \
1062     return(exc); \
1063     }
1064    
1065     /* LHU: Load Half-Word Unsigned */
1066     #define MTS_LHU(X) \
1067     fastcall u_int mts##X##_lhu(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg) \
1068     { \
1069     m_uint64_t data; \
1070     void *haddr; \
1071     u_int exc; \
1072     \
1073     haddr = MTS_ACCESS(X)(cpu,vaddr,MIPS_MEMOP_LHU,2,MTS_READ,&data,&exc); \
1074     if (likely(haddr != NULL)) data = vmtoh16(*(m_uint16_t *)haddr); \
1075     if (likely(!exc)) cpu->gpr[reg] = data & 0xffff; \
1076     return(exc); \
1077     }
1078    
1079     /* LW: Load Word */
1080     #define MTS_LW(X) \
1081     fastcall u_int mts##X##_lw(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg) \
1082     { \
1083     m_uint64_t data; \
1084     void *haddr; \
1085     u_int exc; \
1086     \
1087     haddr = MTS_ACCESS(X)(cpu,vaddr,MIPS_MEMOP_LW,4,MTS_READ,&data,&exc); \
1088     if (likely(haddr != NULL)) data = vmtoh32(*(m_uint32_t *)haddr); \
1089     if (likely(!exc)) cpu->gpr[reg] = sign_extend(data,32); \
1090     return(exc); \
1091     }
1092    
1093     /* LWU: Load Word Unsigned */
1094     #define MTS_LWU(X) \
1095     fastcall u_int mts##X##_lwu(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg) \
1096     { \
1097     m_uint64_t data; \
1098     void *haddr; \
1099     u_int exc; \
1100     \
1101     haddr = MTS_ACCESS(X)(cpu,vaddr,MIPS_MEMOP_LWU,4,MTS_READ,&data,&exc); \
1102     if (likely(haddr != NULL)) data = vmtoh32(*(m_uint32_t *)haddr); \
1103     if (likely(!exc)) cpu->gpr[reg] = data & 0xffffffff; \
1104     return(exc); \
1105     }
1106    
1107     /* LD: Load Double-Word */
1108     #define MTS_LD(X) \
1109     fastcall u_int mts##X##_ld(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg) \
1110     { \
1111     m_uint64_t data; \
1112     void *haddr; \
1113     u_int exc; \
1114     \
1115     haddr = MTS_ACCESS(X)(cpu,vaddr,MIPS_MEMOP_LD,8,MTS_READ,&data,&exc); \
1116     if (likely(haddr != NULL)) data = vmtoh64(*(m_uint64_t *)haddr); \
1117     if (likely(!exc)) cpu->gpr[reg] = data; \
1118     return(exc); \
1119     }
1120    
1121     /* SB: Store Byte */
1122     #define MTS_SB(X) \
1123     fastcall u_int mts##X##_sb(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg) \
1124     { \
1125     m_uint64_t data; \
1126     void *haddr; \
1127     u_int exc; \
1128     \
1129     data = cpu->gpr[reg] & 0xff; \
1130     haddr = MTS_ACCESS(X)(cpu,vaddr,MIPS_MEMOP_SB,1,MTS_WRITE,&data,&exc); \
1131     if (likely(haddr != NULL)) *(m_uint8_t *)haddr = data; \
1132     return(exc); \
1133     }
1134    
1135     /* SH: Store Half-Word */
1136     #define MTS_SH(X) \
1137     fastcall u_int mts##X##_sh(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg) \
1138     { \
1139     m_uint64_t data; \
1140     void *haddr; \
1141     u_int exc; \
1142     \
1143     data = cpu->gpr[reg] & 0xffff; \
1144     haddr = MTS_ACCESS(X)(cpu,vaddr,MIPS_MEMOP_SH,2,MTS_WRITE,&data,&exc); \
1145     if (likely(haddr != NULL)) *(m_uint16_t *)haddr = htovm16(data); \
1146     return(exc); \
1147     }
1148    
1149     /* SW: Store Word */
1150     #define MTS_SW(X) \
1151     fastcall u_int mts##X##_sw(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg) \
1152     { \
1153     m_uint64_t data; \
1154     void *haddr; \
1155     u_int exc; \
1156     \
1157     data = cpu->gpr[reg] & 0xffffffff; \
1158     haddr = MTS_ACCESS(X)(cpu,vaddr,MIPS_MEMOP_SW,4,MTS_WRITE,&data,&exc); \
1159     if (likely(haddr != NULL)) *(m_uint32_t *)haddr = htovm32(data); \
1160     return(exc); \
1161     }
1162    
1163     /* SD: Store Double-Word */
1164     #define MTS_SD(X) \
1165     fastcall u_int mts##X##_sd(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg) \
1166     { \
1167     m_uint64_t data; \
1168     void *haddr; \
1169     u_int exc; \
1170     \
1171     data = cpu->gpr[reg]; \
1172     haddr = MTS_ACCESS(X)(cpu,vaddr,MIPS_MEMOP_SD,8,MTS_WRITE,&data,&exc); \
1173     if (likely(haddr != NULL)) *(m_uint64_t *)haddr = htovm64(data); \
1174     return(exc); \
1175     }
1176    
1177     /* LDC1: Load Double-Word To Coprocessor 1 */
1178     #define MTS_LDC1(X) \
1179     fastcall u_int mts##X##_ldc1(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg) \
1180     { \
1181     m_uint64_t data; \
1182     void *haddr; \
1183     u_int exc; \
1184     \
1185     haddr = MTS_ACCESS(X)(cpu,vaddr,MIPS_MEMOP_LDC1,8,MTS_READ,&data,&exc); \
1186     if (likely(haddr != NULL)) data = vmtoh64(*(m_uint64_t *)haddr); \
1187     if (likely(!exc)) cpu->fpu.reg[reg] = data; \
1188     return(exc); \
1189     }
1190    
1191     /* LWL: Load Word Left */
1192     #define MTS_LWL(X) \
1193     fastcall u_int mts##X##_lwl(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg) \
1194     { \
1195     m_uint64_t r_mask,naddr; \
1196     m_uint64_t data; \
1197     u_int m_shift; \
1198     void *haddr; \
1199     u_int exc; \
1200     \
1201     naddr = vaddr & ~(0x03); \
1202     haddr = MTS_ACCESS(X)(cpu,naddr,MIPS_MEMOP_LWL,4,MTS_READ,&data,&exc); \
1203     \
1204     if (likely(haddr != NULL)) \
1205     data = vmtoh32(*(m_uint32_t *)haddr); \
1206     \
1207     if (likely(!exc)) { \
1208     m_shift = (vaddr & 0x03) << 3; \
1209     r_mask = (1ULL << m_shift) - 1; \
1210     data <<= m_shift; \
1211     \
1212     cpu->gpr[reg] &= r_mask; \
1213     cpu->gpr[reg] |= data; \
1214     cpu->gpr[reg] = sign_extend(cpu->gpr[reg],32); \
1215     } \
1216     return(exc); \
1217     }
1218    
1219     /* LWR: Load Word Right */
1220     #define MTS_LWR(X) \
1221     fastcall u_int mts##X##_lwr(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg) \
1222     { \
1223     m_uint64_t r_mask,naddr; \
1224     m_uint64_t data; \
1225     u_int m_shift; \
1226     void *haddr; \
1227     u_int exc; \
1228     \
1229     naddr = vaddr & ~(0x03); \
1230     haddr = MTS_ACCESS(X)(cpu,naddr,MIPS_MEMOP_LWR,4,MTS_READ,&data,&exc); \
1231     \
1232     if (likely(haddr != NULL)) \
1233     data = vmtoh32(*(m_uint32_t *)haddr); \
1234     \
1235     if (likely(!exc)) { \
1236     m_shift = ((vaddr & 0x03) + 1) << 3; \
1237     r_mask = (1ULL << m_shift) - 1; \
1238     \
1239     data = sign_extend(data >> (32 - m_shift),32); \
1240     r_mask = sign_extend(r_mask,32); \
1241     \
1242     cpu->gpr[reg] &= ~r_mask; \
1243     cpu->gpr[reg] |= data; \
1244     } \
1245     return(exc); \
1246     }
1247    
1248     /* LDL: Load Double-Word Left */
1249     #define MTS_LDL(X) \
1250     fastcall u_int mts##X##_ldl(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg) \
1251     { \
1252     m_uint64_t r_mask,naddr; \
1253     m_uint64_t data; \
1254     u_int m_shift; \
1255     void *haddr; \
1256     u_int exc; \
1257     \
1258     naddr = vaddr & ~(0x07); \
1259     haddr = MTS_ACCESS(X)(cpu,naddr,MIPS_MEMOP_LDL,8,MTS_READ,&data,&exc); \
1260     \
1261     if (likely(haddr != NULL)) \
1262     data = vmtoh64(*(m_uint64_t *)haddr); \
1263     \
1264     if (likely(!exc)) { \
1265     m_shift = (vaddr & 0x07) << 3; \
1266     r_mask = (1ULL << m_shift) - 1; \
1267     data <<= m_shift; \
1268     \
1269     cpu->gpr[reg] &= r_mask; \
1270     cpu->gpr[reg] |= data; \
1271     } \
1272     return(exc); \
1273     }
1274    
1275     /* LDR: Load Double-Word Right */
1276     #define MTS_LDR(X) \
1277     fastcall u_int mts##X##_ldr(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg) \
1278     { \
1279     m_uint64_t r_mask,naddr; \
1280     m_uint64_t data; \
1281     u_int m_shift; \
1282     void *haddr; \
1283     u_int exc; \
1284     \
1285     naddr = vaddr & ~(0x07); \
1286     haddr = MTS_ACCESS(X)(cpu,naddr,MIPS_MEMOP_LDR,8,MTS_READ,&data,&exc); \
1287     \
1288     if (likely(haddr != NULL)) \
1289     data = vmtoh64(*(m_uint64_t *)haddr); \
1290     \
1291     if (likely(!exc)) { \
1292     m_shift = ((vaddr & 0x07) + 1) << 3; \
1293     r_mask = (1ULL << m_shift) - 1; \
1294     data >>= (64 - m_shift); \
1295     \
1296     cpu->gpr[reg] &= ~r_mask; \
1297     cpu->gpr[reg] |= data; \
1298     } \
1299     return(exc); \
1300     }
1301    
1302     /* SWL: Store Word Left */
1303     #define MTS_SWL(X) \
1304     fastcall u_int mts##X##_swl(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg) \
1305     { \
1306     m_uint64_t d_mask,naddr; \
1307     m_uint64_t data; \
1308     u_int r_shift; \
1309     void *haddr; \
1310     u_int exc; \
1311     \
1312     naddr = vaddr & ~(0x03ULL); \
1313     haddr = MTS_ACCESS(X)(cpu,naddr,MIPS_MEMOP_SWL,4,MTS_READ,&data,&exc); \
1314     if (unlikely(exc)) return(exc); \
1315     \
1316     if (likely(haddr != NULL)) \
1317     data = vmtoh32(*(m_uint32_t *)haddr); \
1318     \
1319     r_shift = (vaddr & 0x03) << 3; \
1320     d_mask = 0xffffffff >> r_shift; \
1321     \
1322     data &= ~d_mask; \
1323     data |= (cpu->gpr[reg] & 0xffffffff) >> r_shift; \
1324     \
1325     haddr = MTS_ACCESS(X)(cpu,naddr,MIPS_MEMOP_SWL,4,MTS_WRITE,&data,&exc); \
1326     if (likely(haddr != NULL)) *(m_uint32_t *)haddr = htovm32(data); \
1327     return(exc); \
1328     }
1329    
1330     /* SWR: Store Word Right */
1331     #define MTS_SWR(X) \
1332     fastcall u_int mts##X##_swr(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg) \
1333     { \
1334     m_uint64_t d_mask,naddr; \
1335     m_uint64_t data; \
1336     u_int r_shift; \
1337     void *haddr; \
1338     u_int exc; \
1339     \
1340     naddr = vaddr & ~(0x03); \
1341     haddr = MTS_ACCESS(X)(cpu,naddr,MIPS_MEMOP_SWR,4,MTS_READ,&data,&exc); \
1342     if (unlikely(exc)) return(exc); \
1343     \
1344     if (likely(haddr != NULL)) \
1345     data = vmtoh32(*(m_uint32_t *)haddr); \
1346     \
1347     r_shift = ((vaddr & 0x03) + 1) << 3; \
1348     d_mask = 0xffffffff >> r_shift; \
1349     \
1350     data &= d_mask; \
1351     data |= (cpu->gpr[reg] << (32 - r_shift)) & 0xffffffff; \
1352     \
1353     haddr = MTS_ACCESS(X)(cpu,naddr,MIPS_MEMOP_SWR,4,MTS_WRITE,&data,&exc); \
1354     if (likely(haddr != NULL)) *(m_uint32_t *)haddr = htovm32(data); \
1355     return(exc); \
1356     }
1357    
1358     /* SDL: Store Double-Word Left */
1359     #define MTS_SDL(X) \
1360     fastcall u_int mts##X##_sdl(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg) \
1361     { \
1362     m_uint64_t d_mask,naddr; \
1363     m_uint64_t data; \
1364     u_int r_shift; \
1365     void *haddr; \
1366     u_int exc; \
1367     \
1368     naddr = vaddr & ~(0x07); \
1369     haddr = MTS_ACCESS(X)(cpu,naddr,MIPS_MEMOP_SDL,8,MTS_READ,&data,&exc); \
1370     if (unlikely(exc)) return(exc); \
1371     \
1372     if (likely(haddr != NULL)) \
1373     data = vmtoh64(*(m_uint64_t *)haddr); \
1374     \
1375     r_shift = (vaddr & 0x07) << 3; \
1376     d_mask = 0xffffffffffffffffULL >> r_shift; \
1377     \
1378     data &= ~d_mask; \
1379     data |= cpu->gpr[reg] >> r_shift; \
1380     \
1381     haddr = MTS_ACCESS(X)(cpu,naddr,MIPS_MEMOP_SDL,8,MTS_WRITE,&data,&exc); \
1382     if (likely(haddr != NULL)) *(m_uint64_t *)haddr = htovm64(data); \
1383     return(exc); \
1384     }
1385    
1386     /* SDR: Store Double-Word Right */
1387     #define MTS_SDR(X) \
1388     fastcall u_int mts##X##_sdr(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg) \
1389     { \
1390     m_uint64_t d_mask,naddr; \
1391     m_uint64_t data; \
1392     u_int r_shift; \
1393     void *haddr; \
1394     u_int exc; \
1395     \
1396     naddr = vaddr & ~(0x07); \
1397     haddr = MTS_ACCESS(X)(cpu,naddr,MIPS_MEMOP_SDR,8,MTS_READ,&data,&exc); \
1398     if (unlikely(exc)) return(exc); \
1399     \
1400     if (likely(haddr != NULL)) \
1401     data = vmtoh64(*(m_uint64_t *)haddr); \
1402     \
1403     r_shift = ((vaddr & 0x07) + 1) << 3; \
1404     d_mask = 0xffffffffffffffffULL >> r_shift; \
1405     \
1406     data &= d_mask; \
1407     data |= cpu->gpr[reg] << (64 - r_shift); \
1408     \
1409     haddr = MTS_ACCESS(X)(cpu,naddr,MIPS_MEMOP_SDR,8,MTS_WRITE,&data,&exc); \
1410     if (likely(haddr != NULL)) *(m_uint64_t *)haddr = htovm64(data); \
1411     return(exc); \
1412     }
1413    
1414     /* LL: Load Linked */
1415     #define MTS_LL(X) \
1416     fastcall u_int mts##X##_ll(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg) \
1417     { \
1418     m_uint64_t data; \
1419     void *haddr; \
1420     u_int exc; \
1421     \
1422     haddr = MTS_ACCESS(X)(cpu,vaddr,MIPS_MEMOP_LL,4,MTS_READ,&data,&exc); \
1423     if (likely(haddr != NULL)) data = vmtoh32(*(m_uint32_t *)haddr); \
1424     \
1425     if (likely(!exc)) { \
1426     cpu->gpr[reg] = sign_extend(data,32); \
1427     cpu->ll_bit = 1; \
1428     } \
1429     \
1430     return(exc); \
1431     }
1432    
1433     /* SC: Store Conditional */
1434     #define MTS_SC(X) \
1435     fastcall u_int mts##X##_sc(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg) \
1436     { \
1437     m_uint64_t data; \
1438     void *haddr; \
1439     u_int exc = 0; \
1440     \
1441     if (cpu->ll_bit) { \
1442     data = cpu->gpr[reg] & 0xffffffff; \
1443     haddr = MTS_ACCESS(X)(cpu,vaddr,MIPS_MEMOP_SC,4,MTS_WRITE,&data,&exc); \
1444     if (likely(haddr != NULL)) *(m_uint32_t *)haddr = htovm32(data); \
1445     } \
1446     \
1447     if (likely(!exc)) \
1448     cpu->gpr[reg] = cpu->ll_bit; \
1449     return(exc); \
1450     }
1451    
1452     /* SDC1: Store Double-Word from Coprocessor 1 */
1453     #define MTS_SDC1(X) \
1454     fastcall u_int mts##X##_sdc1(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg) \
1455     { \
1456     m_uint64_t data; \
1457     void *haddr; \
1458     u_int exc; \
1459     \
1460     data = cpu->fpu.reg[reg]; \
1461     haddr = MTS_ACCESS(X)(cpu,vaddr,MIPS_MEMOP_SDC1,8,MTS_WRITE,&data,&exc); \
1462     if (likely(haddr != NULL)) *(m_uint64_t *)haddr = htovm64(data); \
1463     return(exc); \
1464     }
1465    
1466     /* CACHE: Cache operation */
1467     fastcall u_int mts_cache(cpu_mips_t *cpu,m_uint64_t vaddr,u_int op)
1468     {
1469     struct insn_block *block;
1470     m_uint32_t phys_page;
1471    
1472     #if DEBUG_CACHE
1473     cpu_log(cpu,"MTS","CACHE: PC=0x%llx, vaddr=0x%llx, cache=%u, code=%u\n",
1474     cpu->pc, vaddr, op & 0x3, op >> 2);
1475     #endif
1476    
1477     if (!cpu->translate(cpu,vaddr,&phys_page)) {
1478     if ((phys_page < 1048576) && cpu->exec_phys_map) {
1479     block = cpu->exec_phys_map[phys_page];
1480    
1481     if (block) {
1482     if ((cpu->pc < block->start_pc) ||
1483     ((cpu->pc - block->start_pc) >= MIPS_MIN_PAGE_SIZE))
1484     {
1485     #if DEBUG_CACHE
1486     cpu_log(cpu,"MTS",
1487     "CACHE: removing compiled page at 0x%llx, pc=0x%llx\n",
1488     block->start_pc,cpu->pc);
1489     #endif
1490     cpu->exec_phys_map[phys_page] = NULL;
1491     insn_block_free(cpu,block,TRUE);
1492     }
1493     else
1494     {
1495     #if DEBUG_CACHE
1496     cpu_log(cpu,"MTS",
1497     "CACHE: trying to remove page 0x%llx with pc=0x%llx\n",
1498     block->start_pc,cpu->pc);
1499     #endif
1500     }
1501     }
1502     }
1503     }
1504    
1505     return(0);
1506     }
1507    
1508     /*
1509     * "Instanciation" of MIPS Memory Operations.
1510     */
1511     MTS_MEMOP(LB)
1512     MTS_MEMOP(LBU)
1513     MTS_MEMOP(LH)
1514     MTS_MEMOP(LHU)
1515     MTS_MEMOP(LW)
1516     MTS_MEMOP(LWU)
1517     MTS_MEMOP(LD)
1518     MTS_MEMOP(SB)
1519     MTS_MEMOP(SH)
1520     MTS_MEMOP(SW)
1521     MTS_MEMOP(SD)
1522     MTS_MEMOP(LDC1)
1523     MTS_MEMOP(LWL)
1524     MTS_MEMOP(LWR)
1525     MTS_MEMOP(LDL)
1526     MTS_MEMOP(LDR)
1527     MTS_MEMOP(SWL)
1528     MTS_MEMOP(SWR)
1529     MTS_MEMOP(SDL)
1530     MTS_MEMOP(SDR)
1531     MTS_MEMOP(LL)
1532     MTS_MEMOP(SC)
1533     MTS_MEMOP(SDC1)
1534    
1535     /* Fast assembly routines */
1536     #ifdef FAST_ASM
1537     extern fastcall u_int mts32_lw_asm(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg);
1538     #endif
1539    
1540     /* === MTS32 Cache Management ============================================= */
1541    
1542     /* MTS32 map/unmap/rebuild "API" functions */
1543     void mts32_api_map(cpu_mips_t *cpu,m_uint64_t vaddr,m_uint64_t paddr,
1544     m_uint32_t len,int cache_access,int tlb_index)
1545     {
1546     mts32_map(cpu,vaddr,paddr,len,cache_access);
1547     }
1548    
1549     void mts32_api_unmap(cpu_mips_t *cpu,m_uint64_t vaddr,m_uint32_t len,
1550     m_uint32_t val,int tlb_index)
1551     {
1552     mts32_unmap(cpu,vaddr,len,val);
1553     }
1554    
1555     void mts32_api_rebuild(cpu_mips_t *cpu)
1556     {
1557     u_int cpu_mode;
1558    
1559     cpu_mode = cp0_get_mode(cpu);
1560    
1561     /* The complete address space gives AE (address error) */
1562     if (mts32_init_empty(cpu) == -1)
1563     return;
1564    
1565     /* USEG: 0x00000000 -> 0x80000000: 2 GB mapped (TLB) */
1566     mts32_unmap(cpu,MIPS_KUSEG_BASE,MIPS_KUSEG_SIZE,MTS_ACC_T);
1567    
1568     /* If the CPU is in Kernel Mode, activate KSEG segments */
1569     if (cpu_mode == MIPS_CP0_STATUS_KM) {
1570     /* KSEG0 / KSEG1 : physical memory */
1571     mts32_km_map_all_dev(cpu);
1572    
1573     /* KSSEG: 0xc0000000 -> 0xe0000000: 0.5GB mapped (TLB) */
1574     mts32_unmap(cpu,MIPS_KSSEG_BASE,MIPS_KSSEG_SIZE,MTS_ACC_T);
1575    
1576     /* KSEG3: 0xe0000000 -> 0xffffffff: 0.5GB mapped (TLB) */
1577     mts32_unmap(cpu,MIPS_KSEG3_BASE,MIPS_KSEG3_SIZE,MTS_ACC_T);
1578     } else {
1579     if (cpu_mode == MIPS_CP0_STATUS_SM) {
1580     /* SSEG: 0xc0000000 -> 0xe0000000: 0.5GB mapped (TLB) */
1581     mts32_unmap(cpu,MIPS_KSSEG_BASE,MIPS_KSSEG_SIZE,MTS_ACC_T);
1582     }
1583     }
1584    
1585     /* Map all TLB entries */
1586     cp0_map_all_tlb_to_mts(cpu);
1587     }
1588    
1589     /* === MTS64 Cache Management ============================================= */
1590    
1591     /* MTS64 map/unmap/rebuild "API" functions */
1592     void mts64_api_map(cpu_mips_t *cpu,m_uint64_t vaddr,m_uint64_t paddr,
1593     m_uint32_t len,int cache_access,int tlb_index)
1594     {
1595     /* nothing to do, the cache will be filled on-the-fly */
1596     }
1597    
1598     void mts64_api_unmap(cpu_mips_t *cpu,m_uint64_t vaddr,m_uint32_t len,
1599     m_uint32_t val,int tlb_index)
1600     {
1601     /* Invalidate the TLB entry or the full cache if no index is specified */
1602     if (tlb_index != -1)
1603     mts64_invalidate_tlb_entry(cpu,tlb_index);
1604     else
1605     mts64_invalidate_cache(cpu);
1606     }
1607    
1608     void mts64_api_rebuild(cpu_mips_t *cpu)
1609     {
1610     mts64_invalidate_cache(cpu);
1611     }
1612    
1613     /* ======================================================================== */
1614    
1615     /* Initialize memory 32-bit access vectors */
1616     void mts32_init_memop_vectors(cpu_mips_t *cpu)
1617     {
1618     /* XXX TODO:
1619     * - LD/SD forbidden in Supervisor/User modes with 32-bit addresses.
1620     */
1621    
1622     /* API vectors */
1623     cpu->mts_map = mts32_api_map;
1624     cpu->mts_unmap = mts32_api_unmap;
1625     cpu->mts_rebuild = mts32_api_rebuild;
1626    
1627     /* memory lookup operation */
1628     cpu->mem_op_lookup = mts32_lookup;
1629    
1630     /* Load Operations */
1631     cpu->mem_op_fn[MIPS_MEMOP_LB] = mts32_lb;
1632     cpu->mem_op_fn[MIPS_MEMOP_LBU] = mts32_lbu;
1633     cpu->mem_op_fn[MIPS_MEMOP_LH] = mts32_lh;
1634     cpu->mem_op_fn[MIPS_MEMOP_LHU] = mts32_lhu;
1635     cpu->mem_op_fn[MIPS_MEMOP_LW] = mts32_lw;
1636     cpu->mem_op_fn[MIPS_MEMOP_LWU] = mts32_lwu;
1637     cpu->mem_op_fn[MIPS_MEMOP_LD] = mts32_ld;
1638     cpu->mem_op_fn[MIPS_MEMOP_LDL] = mts32_ldl;
1639     cpu->mem_op_fn[MIPS_MEMOP_LDR] = mts32_ldr;
1640    
1641     /* Store Operations */
1642     cpu->mem_op_fn[MIPS_MEMOP_SB] = mts32_sb;
1643     cpu->mem_op_fn[MIPS_MEMOP_SH] = mts32_sh;
1644     cpu->mem_op_fn[MIPS_MEMOP_SW] = mts32_sw;
1645     cpu->mem_op_fn[MIPS_MEMOP_SD] = mts32_sd;
1646    
1647     /* Load Left/Right operations */
1648     cpu->mem_op_fn[MIPS_MEMOP_LWL] = mts32_lwl;
1649     cpu->mem_op_fn[MIPS_MEMOP_LWR] = mts32_lwr;
1650     cpu->mem_op_fn[MIPS_MEMOP_LDL] = mts32_ldl;
1651     cpu->mem_op_fn[MIPS_MEMOP_LDR] = mts32_ldr;
1652    
1653     /* Store Left/Right operations */
1654     cpu->mem_op_fn[MIPS_MEMOP_SWL] = mts32_swl;
1655     cpu->mem_op_fn[MIPS_MEMOP_SWR] = mts32_swr;
1656     cpu->mem_op_fn[MIPS_MEMOP_SDL] = mts32_sdl;
1657     cpu->mem_op_fn[MIPS_MEMOP_SDR] = mts32_sdr;
1658    
1659     /* LL/SC - Load Linked / Store Conditional */
1660     cpu->mem_op_fn[MIPS_MEMOP_LL] = mts32_ll;
1661     cpu->mem_op_fn[MIPS_MEMOP_SC] = mts32_sc;
1662    
1663     /* Coprocessor 1 memory access functions */
1664     cpu->mem_op_fn[MIPS_MEMOP_LDC1] = mts32_ldc1;
1665     cpu->mem_op_fn[MIPS_MEMOP_SDC1] = mts32_sdc1;
1666    
1667     /* Cache Operation */
1668     cpu->mem_op_fn[MIPS_MEMOP_CACHE] = mts_cache;
1669    
1670     #if defined(FAST_ASM) && MTSASM_ENABLE
1671     if (cpu->vm->jit_use)
1672     cpu->mem_op_fn[MIPS_MEMOP_LW] = mts32_lw_asm;
1673     #endif
1674     }
1675    
1676     /* Initialize memory 64-bit access vectors */
1677     void mts64_init_memop_vectors(cpu_mips_t *cpu)
1678     {
1679     /* API vectors */
1680     cpu->mts_map = mts64_api_map;
1681     cpu->mts_unmap = mts64_api_unmap;
1682     cpu->mts_rebuild = mts64_api_rebuild;
1683    
1684     /* memory lookup operation */
1685     cpu->mem_op_lookup = mts64_lookup;
1686    
1687     cpu->translate = mts64_translate;
1688    
1689     /* Load Operations */
1690     cpu->mem_op_fn[MIPS_MEMOP_LB] = mts64_lb;
1691     cpu->mem_op_fn[MIPS_MEMOP_LBU] = mts64_lbu;
1692     cpu->mem_op_fn[MIPS_MEMOP_LH] = mts64_lh;
1693     cpu->mem_op_fn[MIPS_MEMOP_LHU] = mts64_lhu;
1694     cpu->mem_op_fn[MIPS_MEMOP_LW] = mts64_lw;
1695     cpu->mem_op_fn[MIPS_MEMOP_LWU] = mts64_lwu;
1696     cpu->mem_op_fn[MIPS_MEMOP_LD] = mts64_ld;
1697     cpu->mem_op_fn[MIPS_MEMOP_LDL] = mts64_ldl;
1698     cpu->mem_op_fn[MIPS_MEMOP_LDR] = mts64_ldr;
1699    
1700     /* Store Operations */
1701     cpu->mem_op_fn[MIPS_MEMOP_SB] = mts64_sb;
1702     cpu->mem_op_fn[MIPS_MEMOP_SH] = mts64_sh;
1703     cpu->mem_op_fn[MIPS_MEMOP_SW] = mts64_sw;
1704     cpu->mem_op_fn[MIPS_MEMOP_SD] = mts64_sd;
1705    
1706     /* Load Left/Right operations */
1707     cpu->mem_op_fn[MIPS_MEMOP_LWL] = mts64_lwl;
1708     cpu->mem_op_fn[MIPS_MEMOP_LWR] = mts64_lwr;
1709     cpu->mem_op_fn[MIPS_MEMOP_LDL] = mts64_ldl;
1710     cpu->mem_op_fn[MIPS_MEMOP_LDR] = mts64_ldr;
1711    
1712     /* Store Left/Right operations */
1713     cpu->mem_op_fn[MIPS_MEMOP_SWL] = mts64_swl;
1714     cpu->mem_op_fn[MIPS_MEMOP_SWR] = mts64_swr;
1715     cpu->mem_op_fn[MIPS_MEMOP_SDL] = mts64_sdl;
1716     cpu->mem_op_fn[MIPS_MEMOP_SDR] = mts64_sdr;
1717    
1718     /* LL/SC - Load Linked / Store Conditional */
1719     cpu->mem_op_fn[MIPS_MEMOP_LL] = mts64_ll;
1720     cpu->mem_op_fn[MIPS_MEMOP_SC] = mts64_sc;
1721    
1722     /* Coprocessor 1 memory access functions */
1723     cpu->mem_op_fn[MIPS_MEMOP_LDC1] = mts64_ldc1;
1724     cpu->mem_op_fn[MIPS_MEMOP_SDC1] = mts64_sdc1;
1725    
1726     /* Cache Operation */
1727     cpu->mem_op_fn[MIPS_MEMOP_CACHE] = mts_cache;
1728     }
1729    
1730     /* Initialize memory access vectors */
1731     void mts_init_memop_vectors(cpu_mips_t *cpu)
1732     {
1733     /* TEST */
1734     mts64_init_memop_vectors(cpu);
1735     }
1736    
1737     /* Shutdown MTS subsystem */
1738     void mts_shutdown(cpu_mips_t *cpu)
1739     {
1740     mts32_shutdown(cpu);
1741     mts64_shutdown(cpu);
1742     }
1743    
1744     /* Copy a memory block from VM physical RAM to real host */
1745     void physmem_copy_from_vm(vm_instance_t *vm,void *real_buffer,
1746     m_uint64_t paddr,size_t len)
1747     {
1748     struct vdevice *vm_ram;
1749     u_char *ptr;
1750    
1751     if ((vm_ram = dev_lookup(vm,paddr,FALSE)) != NULL) {
1752     assert(vm_ram->host_addr != 0);
1753     ptr = (u_char *)vm_ram->host_addr + (paddr - vm_ram->phys_addr);
1754     memcpy(real_buffer,ptr,len);
1755     }
1756     }
1757    
1758     /* Copy a memory block to VM physical RAM from real host */
1759     void physmem_copy_to_vm(vm_instance_t *vm,void *real_buffer,
1760     m_uint64_t paddr,size_t len)
1761     {
1762     struct vdevice *vm_ram;
1763     u_char *ptr;
1764    
1765     if ((vm_ram = dev_lookup(vm,paddr,FALSE)) != NULL) {
1766     assert(vm_ram->host_addr != 0);
1767     ptr = (u_char *)vm_ram->host_addr + (paddr - vm_ram->phys_addr);
1768     memcpy(ptr,real_buffer,len);
1769     }
1770     }
1771    
1772     /* Copy a 32-bit word from the VM physical RAM to real host */
1773     m_uint32_t physmem_copy_u32_from_vm(vm_instance_t *vm,m_uint64_t paddr)
1774     {
1775     struct vdevice *dev;
1776     m_uint32_t offset;
1777     m_uint64_t tmp;
1778     void *ptr;
1779    
1780     if (unlikely((dev = dev_lookup(vm,paddr,FALSE)) == NULL))
1781     return(0);
1782    
1783     offset = paddr - dev->phys_addr;
1784    
1785     if ((dev->host_addr != 0) && !(dev->flags & VDEVICE_FLAG_NO_MTS_MMAP))
1786     ptr = (u_char *)dev->host_addr + offset;
1787     else {
1788     ptr = dev->handler(vm->boot_cpu,dev,offset,4,MTS_READ,&tmp);
1789     if (!ptr) return(tmp);
1790     }
1791    
1792     return(vmtoh32(*(m_uint32_t *)ptr));
1793     }
1794    
1795     /* Copy a 32-bit word to the VM physical RAM from real host */
1796     void physmem_copy_u32_to_vm(vm_instance_t *vm,m_uint64_t paddr,m_uint32_t val)
1797     {
1798     struct vdevice *dev;
1799     m_uint32_t offset;
1800     m_uint64_t tmp;
1801     void *ptr;
1802    
1803     if (unlikely((dev = dev_lookup(vm,paddr,FALSE)) == NULL))
1804     return;
1805    
1806     offset = paddr - dev->phys_addr;
1807    
1808     if ((dev->host_addr != 0) && !(dev->flags & VDEVICE_FLAG_NO_MTS_MMAP))
1809     ptr = (u_char *)dev->host_addr + offset;
1810     else {
1811     tmp = val;
1812     ptr = dev->handler(vm->boot_cpu,dev,offset,4,MTS_WRITE,&tmp);
1813     if (!ptr) return;
1814     }
1815    
1816     *(m_uint32_t *)ptr = htovm32(val);
1817     }
1818    
1819     /* Copy a 16-bit word from the VM physical RAM to real host */
1820     m_uint16_t physmem_copy_u16_from_vm(vm_instance_t *vm,m_uint64_t paddr)
1821     {
1822     struct vdevice *dev;
1823     m_uint32_t offset;
1824     m_uint64_t tmp;
1825     void *ptr;
1826    
1827     if (unlikely((dev = dev_lookup(vm,paddr,FALSE)) == NULL))
1828     return(0);
1829    
1830     offset = paddr - dev->phys_addr;
1831    
1832     if ((dev->host_addr != 0) && !(dev->flags & VDEVICE_FLAG_NO_MTS_MMAP))
1833     ptr = (u_char *)dev->host_addr + offset;
1834     else {
1835     ptr = dev->handler(vm->boot_cpu,dev,offset,2,MTS_READ,&tmp);
1836     if (!ptr) return(tmp);
1837     }
1838    
1839     return(vmtoh16(*(m_uint16_t *)ptr));
1840     }
1841    
1842     /* Copy a 16-bit word to the VM physical RAM from real host */
1843     void physmem_copy_u16_to_vm(vm_instance_t *vm,m_uint64_t paddr,m_uint16_t val)
1844     {
1845     struct vdevice *dev;
1846     m_uint32_t offset;
1847     m_uint64_t tmp;
1848     void *ptr;
1849    
1850     if (unlikely((dev = dev_lookup(vm,paddr,FALSE)) == NULL))
1851     return;
1852    
1853     offset = paddr - dev->phys_addr;
1854    
1855     if ((dev->host_addr != 0) && !(dev->flags & VDEVICE_FLAG_NO_MTS_MMAP))
1856     ptr = (u_char *)dev->host_addr + offset;
1857     else {
1858     tmp = val;
1859     ptr = dev->handler(vm->boot_cpu,dev,offset,2,MTS_WRITE,&tmp);
1860     if (!ptr) return;
1861     }
1862    
1863     *(m_uint16_t *)ptr = htovm16(val);
1864     }
1865    
1866     /* DMA transfer operation */
1867     void physmem_dma_transfer(vm_instance_t *vm,m_uint64_t src,m_uint64_t dst,
1868     size_t len)
1869     {
1870     struct vdevice *src_dev,*dst_dev;
1871     u_char *sptr,*dptr;
1872    
1873     src_dev = dev_lookup(vm,src,FALSE);
1874     dst_dev = dev_lookup(vm,dst,FALSE);
1875    
1876     if ((src_dev != NULL) && (dst_dev != NULL)) {
1877     assert(src_dev->host_addr != 0);
1878     assert(dst_dev->host_addr != 0);
1879    
1880     sptr = (u_char *)src_dev->host_addr + (src - src_dev->phys_addr);
1881     dptr = (u_char *)dst_dev->host_addr + (dst - dst_dev->phys_addr);
1882     memcpy(dptr,sptr,len);
1883     } else {
1884     vm_log(vm,"DMA","unable to transfer from 0x%llx to 0x%llx (len=%lu)\n",
1885     src,dst,(u_long)len);
1886     }
1887     }
1888    
1889     /* strlen in VM physical memory */
1890     size_t physmem_strlen(vm_instance_t *vm,m_uint64_t paddr)
1891     {
1892     struct vdevice *vm_ram;
1893     size_t len = 0;
1894     char *ptr;
1895    
1896     if ((vm_ram = dev_lookup(vm,paddr,TRUE)) != NULL) {
1897     ptr = (char *)vm_ram->host_addr + (paddr - vm_ram->phys_addr);
1898     len = strlen(ptr);
1899     }
1900    
1901     return(len);
1902     }
1903    
1904     /* Physical memory dump (32-bit words) */
1905     void physmem_dump_vm(vm_instance_t *vm,m_uint64_t paddr,m_uint32_t u32_count)
1906     {
1907     m_uint32_t i;
1908    
1909     for(i=0;i<u32_count;i++) {
1910     vm_log(vm,"physmem_dump","0x%8.8llx: 0x%8.8x\n",
1911     paddr+(i<<2),physmem_copy_u32_from_vm(vm,paddr+(i<<2)));
1912     }
1913     }

  ViewVC Help
Powered by ViewVC 1.1.26