src/cpus/memory_mips_v2p.c

/*
 *  Copyright (C) 2003-2007  Anders Gavare.  All rights reserved.
 *
 *  Redistribution and use in source and binary forms, with or without
 *  modification, are permitted provided that the following conditions are met:
 *
 *  1. Redistributions of source code must retain the above copyright
 *     notice, this list of conditions and the following disclaimer.
 *  2. Redistributions in binary form must reproduce the above copyright  
 *     notice, this list of conditions and the following disclaimer in the 
 *     documentation and/or other materials provided with the distribution.
 *  3. The name of the author may not be used to endorse or promote products
 *     derived from this software without specific prior written permission.
 *
 *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 *  ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 *  ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE   
 *  FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 *  DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 *  OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 *  HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 *  LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 *  OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 *  SUCH DAMAGE.
 *
 *
 *  $Id: memory_mips_v2p.c,v 1.17 2007/05/22 13:05:43 debug Exp $
 */


/*
 *  translate_v2p():
 *
 *  Translate a virtual MIPS address to a physical address, by looking up the
 *  address in the TLB. On failure, an exception is generated (except for the
 *  case when FLAG_NOEXCEPTIONS is used).
 *
 *  Note: This function is long and hairy, it is included several times with
 *        various defines set, to produce more or less optimized versions of
 *        the function for different emulated CPU types.
 *
 *  V2P_MMU3K       Defined for R2000/R3000 emulation. If it is not defined,
 *                  R4000+/MIPS32/MIPS64 is presumed.
 *  V2P_MMU10K      This enables the use of 44 userspace bits, instead of 40.
 *  V2P_MMU4100     VR41xx processors support 1 KB pages, so their page mask
 *                  is slightly different. (The emulator only supports 4 KB
 *                  pages, though.)
 *  V2P_MMU8K       Not yet. (TODO.)
 *
 *
 *  Note:  Unfortunately, the variable name vpn2 is poorly choosen for R2K/R3K,
 *         since it actual contains the vpn.
 *
 *  Return values:
 *      0  Failure
 *      1  Success, the page is readable only
 *      2  Success, the page is read/write
 */
int TRANSLATE_ADDRESS(struct cpu *cpu, uint64_t vaddr,
        uint64_t *return_paddr, int flags)
{
        int writeflag = flags & FLAG_WRITEFLAG? MEM_WRITE : MEM_READ;
        int no_exceptions = flags & FLAG_NOEXCEPTIONS;
        int ksu, use_tlb, status, i;
        uint64_t vaddr_vpn2=0, vaddr_asid=0;
        int exccode, tlb_refill;
        struct mips_coproc *cp0;

#ifdef V2P_MMU3K
        const int x_64 = 0;
        const int n_tlbs = 64;
        const uint32_t pmask = 0xfff;
        uint64_t xuseg_top;             /*  Well, useg actually.  */
#else
#ifdef V2P_MMU10K
        const uint64_t vpn2_mask = ENTRYHI_R_MASK | ENTRYHI_VPN2_MASK_R10K;
        uint64_t xuseg_top = ENTRYHI_VPN2_MASK_R10K | 0x1fffULL;
#else
#ifdef V2P_MMU4100
        const uint64_t vpn2_mask = ENTRYHI_R_MASK | ENTRYHI_VPN2_MASK | 0x1800;
#else
        const uint64_t vpn2_mask = ENTRYHI_R_MASK | ENTRYHI_VPN2_MASK;
#endif
        uint64_t xuseg_top = ENTRYHI_VPN2_MASK | 0x1fffULL;
#endif
        int x_64;       /*  non-zero for 64-bit address space accesses  */
        int pageshift, n_tlbs;
        uint32_t pmask;
#ifdef V2P_MMU4100
        const int pagemask_mask = PAGEMASK_MASK_R4100;
        const int pagemask_shift = PAGEMASK_SHIFT_R4100;
        const int pfn_shift = 10;
#else
        const int pagemask_mask = PAGEMASK_MASK;
        const int pagemask_shift = PAGEMASK_SHIFT;
        const int pfn_shift = 12;
#endif
#endif  /*  !V2P_MMU3K  */


        exccode = -1;
        tlb_refill = 1;

        /*  Cached values:  */
        cp0 = cpu->cd.mips.coproc[0];
        status = cp0->reg[COP0_STATUS];

        /*
         *  MIPS R4000+ and MIPS64 Address Translation:
         *
         *  An address may be in one of the kernel segments, that are directly
         *  mapped to physical addresses, or the address needs to be looked up
         *  in the TLB entries.
         *
         *  KSU: EXL: ERL: X:  Name:   Range:
         *  ---- ---- ---- --  -----   ------
         *
         *   10   0    0    0  useg    0 - 0x7fffffff    (2GB)  (via TLB)
         *   10   0    0    1  xuseg   0 - 0xffffffffff  (1TB)  (via TLB)
         *
         *   01   0    0    0  suseg   0          - 0x7fffffff  (2GB via TLB)
         *   01   0    0    0  ssseg   0xc0000000 - 0xdfffffff  (0.5 GB via TLB)
         *   01   0    0    1  xsuseg  0 - 0xffffffffff         (1TB)  (via TLB)
         *   01   0    0    1  xsseg   0x4000000000000000 - 0x400000ffffffffff
         *                                        (1TB)  (via TLB)
         *   01   0    0    1  csseg   0xffffffffc0000000 - 0xffffffffdfffffff
         *                                        (0.5TB)  (via TLB)
         *
         *   00   x    x    0  kuseg   0 - 0x7fffffff  (2GB)  (via TLB)  (*)
         *   00   x    x    0  kseg0   0x80000000 - 0x9fffffff (0.5GB)
         *                                        unmapped, cached
         *   00   x    x    0  kseg1   0xa0000000 - 0xbfffffff (0.5GB)
         *                                        unmapped, uncached
         *   00   x    x    0  ksseg   0xc0000000 - 0xdfffffff (0.5GB) (via TLB)
         *   00   x    x    0  kseg3   0xe0000000 - 0xffffffff (0.5GB) (via TLB)
         *   00   x    x    1  xksuseg 0 - 0xffffffffff (1TB) (via TLB) (*)
         *   00   x    x    1  xksseg  0x4000000000000000 - 0x400000ffffffffff
         *                                        (1TB)  (via TLB)
         *   00   x    x    1  xkphys  0x8000000000000000 - 0xbfffffffffffffff
         *   00   x    x    1  xkseg   0xc000000000000000 - 0xc00000ff7fffffff
         *   00   x    x    1  ckseg0  0xffffffff80000000 - 0xffffffff9fffffff
         *   00   x    x    1  ckseg1  0xffffffffa0000000 - 0xffffffffbfffffff
         *   00   x    x    1  cksseg  0xffffffffc0000000 - 0xffffffffdfffffff
         *   00   x    x    1  ckseg3  0xffffffffe0000000 - 0xffffffffffffffff
         *                                        like 0x80000000 - 0xffffffff
         *
         *  (*) = if ERL=1 then kuseg is not via TLB, but unmapped,
         *  uncached physical memory.
         *
         *  (KSU==0 or EXL=1 or ERL=1 is enough to use k*seg*.)
         *
         *  An invalid address causes an Address Error.
         *
         *  See chapter 4, page 96, in the R4000 manual for more info!
         */

#ifdef V2P_MMU3K
        if (status & MIPS1_SR_KU_CUR)
                ksu = KSU_USER;
        else
                ksu = KSU_KERNEL;

        /*  These are needed later:  */
        vaddr_asid = cp0->reg[COP0_ENTRYHI] & R2K3K_ENTRYHI_ASID_MASK;
        vaddr_vpn2 = vaddr & R2K3K_ENTRYHI_VPN_MASK;
#else
        /*  kx,sx,ux = 0 for 32-bit addressing, 1 for 64-bit addressing.  */
        ksu = (status & STATUS_KSU_MASK) >> STATUS_KSU_SHIFT;
        if (status & (STATUS_EXL | STATUS_ERL))
                ksu = KSU_KERNEL;

        switch (ksu) {
        case KSU_USER:
                x_64 = status & STATUS_UX;
                break;
        case KSU_KERNEL:
                x_64 = status & STATUS_KX;
                break;
        case KSU_SUPERVISOR:
                x_64 = status & STATUS_SX;
                /*  FALLTHROUGH, since supervisor address spaces are not
                    really implemented yet.  */
        default:fatal("memory_mips_v2p.c: ksu=%i not yet implemented yet\n",
                    ksu);
                exit(1);
        }

        n_tlbs = cpu->cd.mips.cpu_type.nr_of_tlb_entries;

        /*  Having this here suppresses a compiler warning:  */
        pageshift = 12;

        /*  KUSEG: 0x00000000 - 0x7fffffff if ERL = 1 and KSU = kernel:  */
        if (ksu == KSU_KERNEL && (status & STATUS_ERL) &&
            vaddr <= 0x7fffffff) {
                *return_paddr = vaddr & 0x7fffffff;
                return 2;
        }

        /*
         *  XKPHYS: 0x8000000000000000 - 0xbfffffffffffffff
         *
         *  TODO: Is the correct error generated if accessing XKPHYS from
         *        usermode?
         *
         *  TODO: Magic on SGI machines... Cache control, NUMA, etc.:
         *        0x9000000080000000   = disable L2 cache (?)
         *        0x90000000a0000000   = something on IP30?
         *        0x92.... and 0x96... = NUMA on IP27
         */
        if (ksu == KSU_KERNEL && (vaddr & ENTRYHI_R_MASK) == ENTRYHI_R_XKPHYS) {
                *return_paddr = vaddr & (((uint64_t)1 << 44) - 1);
                return 2;
        }

        /*  This is needed later:  */
        vaddr_asid = cp0->reg[COP0_ENTRYHI] & ENTRYHI_ASID;
        /*  vpn2 depends on pagemask, which is not fixed on R4000  */
#endif

        /*  If 32-bit, truncate address and sign extend:  */
        if (x_64 == 0) {
                vaddr = (int32_t) vaddr;
                xuseg_top = 0x7fffffff;
                /*  (Actually useg for R2000/R3000)  */
        }

        if (vaddr <= xuseg_top) {
                use_tlb = 1;
        } else {
                if (ksu == KSU_KERNEL) {
                        /*  kseg0, kseg1:  */
                        if (vaddr >= (uint64_t)0xffffffff80000000ULL &&
                            vaddr <= (uint64_t)0xffffffffbfffffffULL) {
                                *return_paddr = vaddr & 0x1fffffff;
                                return 2;
                        }

                        /*  other segments:  */
                        use_tlb = 1;
                } else {
                        use_tlb = 0;
                }
        }

        if (use_tlb) {
#ifndef V2P_MMU3K
                int odd = 0;
                uint64_t cached_lo1 = 0;
#endif
                int g_bit, v_bit, d_bit;
                uint64_t cached_hi, cached_lo0;
                uint64_t entry_vpn2 = 0, entry_asid, pfn;
                int i_end;

                i = cpu->cd.mips.last_written_tlb_index;
                i_end = i == 0? n_tlbs-1 : i - 1;

                /*  Scan all TLB entries:  */
                for (;;) {
#ifdef V2P_MMU3K
                        /*  R3000 or similar:  */
                        cached_hi = cp0->tlbs[i].hi;
                        cached_lo0 = cp0->tlbs[i].lo0;

                        entry_vpn2 = cached_hi & R2K3K_ENTRYHI_VPN_MASK;
                        entry_asid = cached_hi & R2K3K_ENTRYHI_ASID_MASK;
                        g_bit = cached_lo0 & R2K3K_ENTRYLO_G;
                        v_bit = cached_lo0 & R2K3K_ENTRYLO_V;
                        d_bit = cached_lo0 & R2K3K_ENTRYLO_D;
#else
                        /*  R4000 or similar:  */
                        pmask = cp0->tlbs[i].mask & pagemask_mask;
                        cached_hi = cp0->tlbs[i].hi;
                        cached_lo0 = cp0->tlbs[i].lo0;
                        cached_lo1 = cp0->tlbs[i].lo1;

                        /*  Optimized for minimum page size:  */
                        if (pmask == 0) {
                                pageshift = pagemask_shift - 1;
                                entry_vpn2 = (cached_hi & vpn2_mask)
                                    >> pagemask_shift;
                                vaddr_vpn2 = (vaddr & vpn2_mask)
                                    >> pagemask_shift;
                                pmask = (1 << (pagemask_shift-1)) - 1;
                                odd = (vaddr >> (pagemask_shift-1)) & 1;
                        } else {
                                /*  Non-standard page mask:  */
                                switch (pmask | ((1 << pagemask_shift) - 1)) {
                                case 0x00007ff: pageshift = 10; break;
                                case 0x0001fff: pageshift = 12; break;
                                case 0x0007fff: pageshift = 14; break;
                                case 0x001ffff: pageshift = 16; break;
                                case 0x007ffff: pageshift = 18; break;
                                case 0x01fffff: pageshift = 20; break;
                                case 0x07fffff: pageshift = 22; break;
                                case 0x1ffffff: pageshift = 24; break;
                                case 0x7ffffff: pageshift = 26; break;
                                default:fatal("pmask=%08"PRIx32"\n", pmask);
                                        exit(1);
                                }

                                entry_vpn2 = (cached_hi &
                                    vpn2_mask) >> (pageshift + 1);
                                vaddr_vpn2 = (vaddr & vpn2_mask) >>
                                    (pageshift + 1);
                                pmask = (1 << pageshift) - 1;
                                odd = (vaddr >> pageshift) & 1;
                        }

                        /*  Assume even virtual page...  */
                        v_bit = cached_lo0 & ENTRYLO_V;
                        d_bit = cached_lo0 & ENTRYLO_D;

#ifdef V2P_MMU8K
                        /*
                         *  TODO:  I don't really know anything about the R8000.
                         *  http://futuretech.mirror.vuurwerk.net/i2sec7.html
                         *  says that it has a three-way associative TLB with
                         *  384 entries, 16KB page size, and some other things.
                         *
                         *  It feels like things like the valid bit (ala R4000)
                         *  and dirty bit are not implemented the same on R8000.
                         *
                         *  http://sgistuff.tastensuppe.de/documents/
                         *              R8000_chipset.html
                         *  also has some info, but no details.
                         */
                        v_bit = 1;      /*  Big TODO  */
                        d_bit = 1;
#endif

                        entry_asid = cached_hi & ENTRYHI_ASID;

                        /*  ... reload pfn, v_bit, d_bit if
                            it was the odd virtual page:  */
                        if (odd) {
                                v_bit = cached_lo1 & ENTRYLO_V;
                                d_bit = cached_lo1 & ENTRYLO_D;
                        }
#ifdef V2P_MMU4100
                        g_bit = cached_lo1 & cached_lo0 & ENTRYLO_G;
#else
                        g_bit = cached_hi & TLB_G;
#endif

#endif

                        /*  Is there a VPN and ASID match?  */
                        if (entry_vpn2 == vaddr_vpn2 &&
                            (entry_asid == vaddr_asid || g_bit)) {
                                /*  debug("OK MAP 1, i=%i { vaddr=%016"PRIx64" "
                                    "==> paddr %016"PRIx64" v=%i d=%i "
                                    "asid=0x%02x }\n", i, (uint64_t) vaddr,
                                    (uint64_t) *return_paddr, v_bit?1:0,
                                    d_bit?1:0, vaddr_asid);  */
                                if (v_bit) {
                                        if (d_bit || (!d_bit &&
                                            writeflag == MEM_READ)) {
                                                uint64_t paddr;
                                                /*  debug("OK MAP 2!!! { w=%i "
                                                    "vaddr=%016"PRIx64" ==> "
                                                    "d=%i v=%i paddr %016"
                                                    PRIx64" ",
                                                    writeflag, (uint64_t)vaddr,
                                                    d_bit?1:0, v_bit?1:0,
                                                    (uint64_t) *return_paddr);
                                                    debug(", tlb entry %2i: ma"
                                                    "sk=%016"PRIx64" hi=%016"
                                                    PRIx64" lo0=%016"PRIx64
                                                    " lo1=%016"PRIx64"\n",
                                                    i, cp0->tlbs[i].mask, cp0->
                                                    tlbs[i].hi, cp0->tlbs[i].
                                                    lo0, cp0->tlbs[i].lo1);
                                                */
#ifdef V2P_MMU3K
                                                pfn = cached_lo0 &
                                                    R2K3K_ENTRYLO_PFN_MASK;
                                                paddr = pfn | (vaddr & pmask);
#else
                                                pfn = ((odd? cached_lo1 :
                                                    cached_lo0)
                                                    & ENTRYLO_PFN_MASK)
                                                    >> ENTRYLO_PFN_SHIFT;
                                                paddr = ((pfn << pfn_shift) &
                                                    ~pmask) | (vaddr & pmask);
#endif

                                                *return_paddr = paddr;
                                                return d_bit? 2 : 1;
                                        } else {
                                                /*  TLB modif. exception  */
                                                tlb_refill = 0;
                                                exccode = EXCEPTION_MOD;
                                                goto exception;
                                        }
                                } else {
                                        /*  TLB invalid exception  */
                                        tlb_refill = 0;
                                        goto exception;
                                }
                        }

                        if (i == i_end)
                                break;

                        /*  Go to the next TLB entry:  */
                        i ++;
                        if (i == n_tlbs)
                                i = 0;
                }
        }

        /*
         *  We are here if for example userland code tried to access
         *  kernel memory, OR if there was a TLB refill.
         */

        if (!use_tlb) {
                tlb_refill = 0;
                if (writeflag == MEM_WRITE)
                        exccode = EXCEPTION_ADES;
                else
                        exccode = EXCEPTION_ADEL;
        }

exception:
        if (no_exceptions)
                return 0;

        /*  TLB Load or Store exception:  */
        if (exccode == -1) {
                if (writeflag == MEM_WRITE)
                        exccode = EXCEPTION_TLBS;
                else
                        exccode = EXCEPTION_TLBL;
        }

#ifdef V2P_MMU3K
        vaddr_asid >>= R2K3K_ENTRYHI_ASID_SHIFT;
        vaddr_vpn2 >>= 12;
#endif

        mips_cpu_exception(cpu, exccode, tlb_refill, vaddr,
            0, vaddr_vpn2, vaddr_asid, x_64);

        /*  Return failure:  */
        return 0;
}

1	dpavlin	14	/*
2	dpavlin	34	* Copyright (C) 2003-2007 Anders Gavare. All rights reserved.
3	dpavlin	14	*
4			* Redistribution and use in source and binary forms, with or without
5			* modification, are permitted provided that the following conditions are met:
6			*
7			* 1. Redistributions of source code must retain the above copyright
8			* notice, this list of conditions and the following disclaimer.
9			* 2. Redistributions in binary form must reproduce the above copyright
10			* notice, this list of conditions and the following disclaimer in the
11			* documentation and/or other materials provided with the distribution.
12			* 3. The name of the author may not be used to endorse or promote products
13			* derived from this software without specific prior written permission.
14			*
15			* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16			* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17			* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18			* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19			* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20			* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21			* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22			* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23			* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24			* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25			* SUCH DAMAGE.
26			*
27			*
28	dpavlin	42	* $Id: memory_mips_v2p.c,v 1.17 2007/05/22 13:05:43 debug Exp $
29	dpavlin	14	*/
30
31
32			/*
33	dpavlin	26	* translate_v2p():
34	dpavlin	14	*
35	dpavlin	32	* Translate a virtual MIPS address to a physical address, by looking up the
36			* address in the TLB. On failure, an exception is generated (except for the
37			* case when FLAG_NOEXCEPTIONS is used).
38	dpavlin	14	*
39	dpavlin	32	* Note: This function is long and hairy, it is included several times with
40			* various defines set, to produce more or less optimized versions of
41			* the function for different emulated CPU types.
42	dpavlin	14	*
43	dpavlin	32	* V2P_MMU3K Defined for R2000/R3000 emulation. If it is not defined,
44			* R4000+/MIPS32/MIPS64 is presumed.
45			* V2P_MMU10K This enables the use of 44 userspace bits, instead of 40.
46			* V2P_MMU4100 VR41xx processors support 1 KB pages, so their page mask
47			* is slightly different. (The emulator only supports 4 KB
48			* pages, though.)
49			* V2P_MMU8K Not yet. (TODO.)
50			*
51			*
52			* Note: Unfortunately, the variable name vpn2 is poorly choosen for R2K/R3K,
53			* since it actual contains the vpn.
54			*
55	dpavlin	14	* Return values:
56			* 0 Failure
57			* 1 Success, the page is readable only
58			* 2 Success, the page is read/write
59			*/
60			int TRANSLATE_ADDRESS(struct cpu *cpu, uint64_t vaddr,
61	dpavlin	26	uint64_t *return_paddr, int flags)
62	dpavlin	14	{
63			int writeflag = flags & FLAG_WRITEFLAG? MEM_WRITE : MEM_READ;
64			int no_exceptions = flags & FLAG_NOEXCEPTIONS;
65			int ksu, use_tlb, status, i;
66			uint64_t vaddr_vpn2=0, vaddr_asid=0;
67			int exccode, tlb_refill;
68			struct mips_coproc *cp0;
69
70			#ifdef V2P_MMU3K
71			const int x_64 = 0;
72			const int n_tlbs = 64;
73	dpavlin	42	const uint32_t pmask = 0xfff;
74	dpavlin	32	uint64_t xuseg_top; /* Well, useg actually. */
75	dpavlin	14	#else
76			#ifdef V2P_MMU10K
77	dpavlin	32	const uint64_t vpn2_mask = ENTRYHI_R_MASK \| ENTRYHI_VPN2_MASK_R10K;
78			uint64_t xuseg_top = ENTRYHI_VPN2_MASK_R10K \| 0x1fffULL;
79	dpavlin	14	#else
80			#ifdef V2P_MMU4100
81	dpavlin	32	const uint64_t vpn2_mask = ENTRYHI_R_MASK \| ENTRYHI_VPN2_MASK \| 0x1800;
82	dpavlin	14	#else
83	dpavlin	32	const uint64_t vpn2_mask = ENTRYHI_R_MASK \| ENTRYHI_VPN2_MASK;
84	dpavlin	14	#endif
85	dpavlin	32	uint64_t xuseg_top = ENTRYHI_VPN2_MASK \| 0x1fffULL;
86	dpavlin	14	#endif
87			int x_64; /* non-zero for 64-bit address space accesses */
88			int pageshift, n_tlbs;
89	dpavlin	42	uint32_t pmask;
90	dpavlin	14	#ifdef V2P_MMU4100
91			const int pagemask_mask = PAGEMASK_MASK_R4100;
92			const int pagemask_shift = PAGEMASK_SHIFT_R4100;
93			const int pfn_shift = 10;
94			#else
95			const int pagemask_mask = PAGEMASK_MASK;
96			const int pagemask_shift = PAGEMASK_SHIFT;
97			const int pfn_shift = 12;
98			#endif
99			#endif /* !V2P_MMU3K */
100
101
102			exccode = -1;
103			tlb_refill = 1;
104
105			/* Cached values: */
106			cp0 = cpu->cd.mips.coproc[0];
107			status = cp0->reg[COP0_STATUS];
108
109			/*
110	dpavlin	32	* MIPS R4000+ and MIPS64 Address Translation:
111	dpavlin	14	*
112	dpavlin	32	* An address may be in one of the kernel segments, that are directly
113			* mapped to physical addresses, or the address needs to be looked up
114			* in the TLB entries.
115	dpavlin	14	*
116			* KSU: EXL: ERL: X: Name: Range:
117			* ---- ---- ---- -- ----- ------
118			*
119			* 10 0 0 0 useg 0 - 0x7fffffff (2GB) (via TLB)
120			* 10 0 0 1 xuseg 0 - 0xffffffffff (1TB) (via TLB)
121			*
122			* 01 0 0 0 suseg 0 - 0x7fffffff (2GB via TLB)
123			* 01 0 0 0 ssseg 0xc0000000 - 0xdfffffff (0.5 GB via TLB)
124			* 01 0 0 1 xsuseg 0 - 0xffffffffff (1TB) (via TLB)
125			* 01 0 0 1 xsseg 0x4000000000000000 - 0x400000ffffffffff
126			* (1TB) (via TLB)
127			* 01 0 0 1 csseg 0xffffffffc0000000 - 0xffffffffdfffffff
128			* (0.5TB) (via TLB)
129			*
130			* 00 x x 0 kuseg 0 - 0x7fffffff (2GB) (via TLB) (*)
131			* 00 x x 0 kseg0 0x80000000 - 0x9fffffff (0.5GB)
132			* unmapped, cached
133			* 00 x x 0 kseg1 0xa0000000 - 0xbfffffff (0.5GB)
134			* unmapped, uncached
135	dpavlin	32	* 00 x x 0 ksseg 0xc0000000 - 0xdfffffff (0.5GB) (via TLB)
136			* 00 x x 0 kseg3 0xe0000000 - 0xffffffff (0.5GB) (via TLB)
137	dpavlin	14	* 00 x x 1 xksuseg 0 - 0xffffffffff (1TB) (via TLB) (*)
138			* 00 x x 1 xksseg 0x4000000000000000 - 0x400000ffffffffff
139			* (1TB) (via TLB)
140			* 00 x x 1 xkphys 0x8000000000000000 - 0xbfffffffffffffff
141			* 00 x x 1 xkseg 0xc000000000000000 - 0xc00000ff7fffffff
142			* 00 x x 1 ckseg0 0xffffffff80000000 - 0xffffffff9fffffff
143			* 00 x x 1 ckseg1 0xffffffffa0000000 - 0xffffffffbfffffff
144			* 00 x x 1 cksseg 0xffffffffc0000000 - 0xffffffffdfffffff
145			* 00 x x 1 ckseg3 0xffffffffe0000000 - 0xffffffffffffffff
146	dpavlin	32	* like 0x80000000 - 0xffffffff
147	dpavlin	14	*
148			* (*) = if ERL=1 then kuseg is not via TLB, but unmapped,
149			* uncached physical memory.
150			*
151			* (KSU==0 or EXL=1 or ERL=1 is enough to use kseg.)
152			*
153			* An invalid address causes an Address Error.
154			*
155			* See chapter 4, page 96, in the R4000 manual for more info!
156			*/
157
158			#ifdef V2P_MMU3K
159			if (status & MIPS1_SR_KU_CUR)
160			ksu = KSU_USER;
161			else
162			ksu = KSU_KERNEL;
163
164			/* These are needed later: */
165			vaddr_asid = cp0->reg[COP0_ENTRYHI] & R2K3K_ENTRYHI_ASID_MASK;
166			vaddr_vpn2 = vaddr & R2K3K_ENTRYHI_VPN_MASK;
167			#else
168	dpavlin	32	/* kx,sx,ux = 0 for 32-bit addressing, 1 for 64-bit addressing. */
169	dpavlin	14	ksu = (status & STATUS_KSU_MASK) >> STATUS_KSU_SHIFT;
170			if (status & (STATUS_EXL \| STATUS_ERL))
171			ksu = KSU_KERNEL;
172
173	dpavlin	32	switch (ksu) {
174			case KSU_USER:
175			x_64 = status & STATUS_UX;
176			break;
177			case KSU_KERNEL:
178	dpavlin	14	x_64 = status & STATUS_KX;
179	dpavlin	32	break;
180			case KSU_SUPERVISOR:
181	dpavlin	14	x_64 = status & STATUS_SX;
182	dpavlin	32	/* FALLTHROUGH, since supervisor address spaces are not
183			really implemented yet. */
184			default:fatal("memory_mips_v2p.c: ksu=%i not yet implemented yet\n",
185			ksu);
186			exit(1);
187			}
188	dpavlin	14
189	dpavlin	32	n_tlbs = cpu->cd.mips.cpu_type.nr_of_tlb_entries;
190
191			/* Having this here suppresses a compiler warning: */
192	dpavlin	14	pageshift = 12;
193
194	dpavlin	32	/* KUSEG: 0x00000000 - 0x7fffffff if ERL = 1 and KSU = kernel: */
195			if (ksu == KSU_KERNEL && (status & STATUS_ERL) &&
196			vaddr <= 0x7fffffff) {
197			*return_paddr = vaddr & 0x7fffffff;
198			return 2;
199			}
200
201	dpavlin	14	/*
202	dpavlin	32	* XKPHYS: 0x8000000000000000 - 0xbfffffffffffffff
203	dpavlin	14	*
204	dpavlin	32	* TODO: Is the correct error generated if accessing XKPHYS from
205			* usermode?
206	dpavlin	14	*
207	dpavlin	32	* TODO: Magic on SGI machines... Cache control, NUMA, etc.:
208			* 0x9000000080000000 = disable L2 cache (?)
209			* 0x90000000a0000000 = something on IP30?
210			* 0x92.... and 0x96... = NUMA on IP27
211	dpavlin	14	*/
212	dpavlin	32	if (ksu == KSU_KERNEL && (vaddr & ENTRYHI_R_MASK) == ENTRYHI_R_XKPHYS) {
213	dpavlin	26	*return_paddr = vaddr & (((uint64_t)1 << 44) - 1);
214	dpavlin	14	return 2;
215			}
216
217			/* This is needed later: */
218			vaddr_asid = cp0->reg[COP0_ENTRYHI] & ENTRYHI_ASID;
219			/* vpn2 depends on pagemask, which is not fixed on R4000 */
220			#endif
221
222	dpavlin	32	/* If 32-bit, truncate address and sign extend: */
223			if (x_64 == 0) {
224			vaddr = (int32_t) vaddr;
225			xuseg_top = 0x7fffffff;
226			/* (Actually useg for R2000/R3000) */
227			}
228	dpavlin	14
229	dpavlin	32	if (vaddr <= xuseg_top) {
230	dpavlin	14	use_tlb = 1;
231	dpavlin	32	} else {
232	dpavlin	14	if (ksu == KSU_KERNEL) {
233			/* kseg0, kseg1: */
234			if (vaddr >= (uint64_t)0xffffffff80000000ULL &&
235			vaddr <= (uint64_t)0xffffffffbfffffffULL) {
236	dpavlin	26	*return_paddr = vaddr & 0x1fffffff;
237	dpavlin	14	return 2;
238			}
239
240			/* other segments: */
241			use_tlb = 1;
242	dpavlin	32	} else {
243	dpavlin	14	use_tlb = 0;
244	dpavlin	32	}
245	dpavlin	14	}
246
247			if (use_tlb) {
248			#ifndef V2P_MMU3K
249	dpavlin	22	int odd = 0;
250			uint64_t cached_lo1 = 0;
251	dpavlin	14	#endif
252			int g_bit, v_bit, d_bit;
253			uint64_t cached_hi, cached_lo0;
254			uint64_t entry_vpn2 = 0, entry_asid, pfn;
255	dpavlin	32	int i_end;
256	dpavlin	14
257	dpavlin	32	i = cpu->cd.mips.last_written_tlb_index;
258			i_end = i == 0? n_tlbs-1 : i - 1;
259
260			/* Scan all TLB entries: */
261			for (;;) {
262	dpavlin	14	#ifdef V2P_MMU3K
263			/* R3000 or similar: */
264			cached_hi = cp0->tlbs[i].hi;
265			cached_lo0 = cp0->tlbs[i].lo0;
266
267			entry_vpn2 = cached_hi & R2K3K_ENTRYHI_VPN_MASK;
268			entry_asid = cached_hi & R2K3K_ENTRYHI_ASID_MASK;
269			g_bit = cached_lo0 & R2K3K_ENTRYLO_G;
270			v_bit = cached_lo0 & R2K3K_ENTRYLO_V;
271			d_bit = cached_lo0 & R2K3K_ENTRYLO_D;
272			#else
273			/* R4000 or similar: */
274			pmask = cp0->tlbs[i].mask & pagemask_mask;
275			cached_hi = cp0->tlbs[i].hi;
276			cached_lo0 = cp0->tlbs[i].lo0;
277			cached_lo1 = cp0->tlbs[i].lo1;
278
279			/* Optimized for minimum page size: */
280			if (pmask == 0) {
281			pageshift = pagemask_shift - 1;
282			entry_vpn2 = (cached_hi & vpn2_mask)
283			>> pagemask_shift;
284			vaddr_vpn2 = (vaddr & vpn2_mask)
285			>> pagemask_shift;
286			pmask = (1 << (pagemask_shift-1)) - 1;
287			odd = (vaddr >> (pagemask_shift-1)) & 1;
288			} else {
289			/* Non-standard page mask: */
290			switch (pmask \| ((1 << pagemask_shift) - 1)) {
291			case 0x00007ff: pageshift = 10; break;
292			case 0x0001fff: pageshift = 12; break;
293			case 0x0007fff: pageshift = 14; break;
294			case 0x001ffff: pageshift = 16; break;
295			case 0x007ffff: pageshift = 18; break;
296			case 0x01fffff: pageshift = 20; break;
297			case 0x07fffff: pageshift = 22; break;
298			case 0x1ffffff: pageshift = 24; break;
299			case 0x7ffffff: pageshift = 26; break;
300	dpavlin	42	default:fatal("pmask=%08"PRIx32"\n", pmask);
301	dpavlin	14	exit(1);
302			}
303
304			entry_vpn2 = (cached_hi &
305			vpn2_mask) >> (pageshift + 1);
306			vaddr_vpn2 = (vaddr & vpn2_mask) >>
307			(pageshift + 1);
308			pmask = (1 << pageshift) - 1;
309			odd = (vaddr >> pageshift) & 1;
310			}
311
312			/* Assume even virtual page... */
313			v_bit = cached_lo0 & ENTRYLO_V;
314			d_bit = cached_lo0 & ENTRYLO_D;
315
316			#ifdef V2P_MMU8K
317			/*
318			* TODO: I don't really know anything about the R8000.
319			* http://futuretech.mirror.vuurwerk.net/i2sec7.html
320			* says that it has a three-way associative TLB with
321			* 384 entries, 16KB page size, and some other things.
322			*
323			* It feels like things like the valid bit (ala R4000)
324			* and dirty bit are not implemented the same on R8000.
325			*
326			* http://sgistuff.tastensuppe.de/documents/
327			* R8000_chipset.html
328			* also has some info, but no details.
329			*/
330			v_bit = 1; /* Big TODO */
331			d_bit = 1;
332			#endif
333
334			entry_asid = cached_hi & ENTRYHI_ASID;
335
336			/* ... reload pfn, v_bit, d_bit if
337			it was the odd virtual page: */
338			if (odd) {
339			v_bit = cached_lo1 & ENTRYLO_V;
340			d_bit = cached_lo1 & ENTRYLO_D;
341			}
342			#ifdef V2P_MMU4100
343			g_bit = cached_lo1 & cached_lo0 & ENTRYLO_G;
344			#else
345			g_bit = cached_hi & TLB_G;
346			#endif
347
348			#endif
349
350			/* Is there a VPN and ASID match? */
351			if (entry_vpn2 == vaddr_vpn2 &&
352			(entry_asid == vaddr_asid \|\| g_bit)) {
353	dpavlin	24	/* debug("OK MAP 1, i=%i { vaddr=%016"PRIx64" "
354			"==> paddr %016"PRIx64" v=%i d=%i "
355			"asid=0x%02x }\n", i, (uint64_t) vaddr,
356	dpavlin	26	(uint64_t) *return_paddr, v_bit?1:0,
357	dpavlin	14	d_bit?1:0, vaddr_asid); */
358			if (v_bit) {
359			if (d_bit \|\| (!d_bit &&
360			writeflag == MEM_READ)) {
361			uint64_t paddr;
362			/* debug("OK MAP 2!!! { w=%i "
363	dpavlin	24	"vaddr=%016"PRIx64" ==> "
364			"d=%i v=%i paddr %016"
365			PRIx64" ",
366			writeflag, (uint64_t)vaddr,
367	dpavlin	14	d_bit?1:0, v_bit?1:0,
368	dpavlin	26	(uint64_t) *return_paddr);
369	dpavlin	14	debug(", tlb entry %2i: ma"
370	dpavlin	24	"sk=%016"PRIx64" hi=%016"
371			PRIx64" lo0=%016"PRIx64
372			" lo1=%016"PRIx64"\n",
373	dpavlin	14	i, cp0->tlbs[i].mask, cp0->
374			tlbs[i].hi, cp0->tlbs[i].
375			lo0, cp0->tlbs[i].lo1);
376			*/
377			#ifdef V2P_MMU3K
378			pfn = cached_lo0 &
379			R2K3K_ENTRYLO_PFN_MASK;
380			paddr = pfn \| (vaddr & pmask);
381			#else
382			pfn = ((odd? cached_lo1 :
383			cached_lo0)
384			& ENTRYLO_PFN_MASK)
385			>> ENTRYLO_PFN_SHIFT;
386	dpavlin	40	paddr = ((pfn << pfn_shift) &
387			~pmask) \| (vaddr & pmask);
388	dpavlin	14	#endif
389
390	dpavlin	26	*return_paddr = paddr;
391	dpavlin	14	return d_bit? 2 : 1;
392			} else {
393			/* TLB modif. exception */
394			tlb_refill = 0;
395			exccode = EXCEPTION_MOD;
396			goto exception;
397			}
398			} else {
399			/* TLB invalid exception */
400			tlb_refill = 0;
401			goto exception;
402			}
403			}
404	dpavlin	32
405			if (i == i_end)
406			break;
407
408			/* Go to the next TLB entry: */
409			i ++;
410			if (i == n_tlbs)
411			i = 0;
412	dpavlin	14	}
413			}
414
415			/*
416			* We are here if for example userland code tried to access
417			* kernel memory, OR if there was a TLB refill.
418			*/
419
420			if (!use_tlb) {
421			tlb_refill = 0;
422			if (writeflag == MEM_WRITE)
423			exccode = EXCEPTION_ADES;
424			else
425			exccode = EXCEPTION_ADEL;
426			}
427
428			exception:
429			if (no_exceptions)
430			return 0;
431
432			/* TLB Load or Store exception: */
433			if (exccode == -1) {
434			if (writeflag == MEM_WRITE)
435			exccode = EXCEPTION_TLBS;
436			else
437			exccode = EXCEPTION_TLBL;
438			}
439
440			#ifdef V2P_MMU3K
441			vaddr_asid >>= R2K3K_ENTRYHI_ASID_SHIFT;
442			vaddr_vpn2 >>= 12;
443			#endif
444
445			mips_cpu_exception(cpu, exccode, tlb_refill, vaddr,
446			0, vaddr_vpn2, vaddr_asid, x_64);
447
448			/* Return failure: */
449			return 0;
450			}
451