cpu/cpu_jitc_x86/jitc.cc

/*
 *      PearPC
 *      jitc.cc
 *
 *      Copyright (C) 2004 Sebastian Biallas (sb@biallas.net)
 *
 *      This program is free software; you can redistribute it and/or modify
 *      it under the terms of the GNU General Public License version 2 as
 *      published by the Free Software Foundation.
 *
 *      This program is distributed in the hope that it will be useful,
 *      but WITHOUT ANY WARRANTY; without even the implied warranty of
 *      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *      GNU General Public License for more details.
 *
 *      You should have received a copy of the GNU General Public License
 *      along with this program; if not, write to the Free Software
 *      Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */

#include <cstdlib>
#include <cstring>

#include "system/sys.h"
#include "tools/snprintf.h"

#include "jitc.h"
#include "jitc_debug.h"
#include "jitc_asm.h"

#include "ppc_dec.h"
#include "ppc_mmu.h"
#include "ppc_tools.h"

JITC gJITC;

static TranslationCacheFragment *jitcAllocFragment();

/*
 *      Intern
 *      Called whenever a new fragment is needed
 *      returns true if a new fragment was really necessary
 */
static bool FASTCALL jitcEmitNextFragment()
{
        // save old
        TranslationCacheFragment *tcf_old = gJITC.currentPage->tcf_current;
        NativeAddress tcp_old = gJITC.currentPage->tcp;
        // alloc new
        gJITC.currentPage->tcf_current = jitcAllocFragment();
        gJITC.currentPage->tcf_current->prev = tcf_old;
        if (((uint)(gJITC.currentPage->tcf_current->base - gJITC.currentPage->tcp)) < 20) {
                // next Fragment directly follows
                gJITC.currentPage->bytesLeft += FRAGMENT_SIZE;
                return false;
        } else {
                gJITC.currentPage->tcp = gJITC.currentPage->tcf_current->base;
                gJITC.currentPage->bytesLeft = FRAGMENT_SIZE;
                // hardcoded JMP from old to new fragment
                // FIXME: use 0xeb if possible
                tcp_old[0] = 0xe9;
                *((uint32 *)&tcp_old[1]) = gJITC.currentPage->tcp - (tcp_old+5);
                return true;
        }
}

/*
 *      emit one byte of native code
 */
void FASTCALL jitcEmit1(byte b)
{
        jitcDebugLogEmit(&b, 1);
        /*
         *      We always have to leave at least 5 bytes in the fragment
         *      to issue a final JMP
         */
        if (gJITC.currentPage->bytesLeft <= 5) {
                jitcEmitNextFragment();
        }
        *(gJITC.currentPage->tcp++) = b;
        gJITC.currentPage->bytesLeft--;
}

/*
 *      emit native code
 */
void FASTCALL jitcEmit(byte *instr, int size)
{
        jitcDebugLogEmit(instr, size);
        if ((gJITC.currentPage->bytesLeft - size) < 5) {
                jitcEmitNextFragment();
        }
        memcpy(gJITC.currentPage->tcp, instr, size);
        gJITC.currentPage->tcp += size;
        gJITC.currentPage->bytesLeft -= size;
}

/*
 *      Assures that the next instruction will be
 *      emitted in the current fragment
 */
bool FASTCALL jitcEmitAssure(int size)
{
        if ((gJITC.currentPage->bytesLeft - size) < 5) {
                jitcEmitNextFragment();
                return false;
        }
        return true;
}

void FASTCALL jitcEmitAlign(int align)
{
        do {
                int missalign = ((uint)gJITC.currentPage->tcp) % align;
                if (missalign) {
                        int bytes = align - missalign;
                        if ((gJITC.currentPage->bytesLeft - bytes) < 5) {
                                if (jitcEmitNextFragment()) continue;
                        }
                        gJITC.currentPage->tcp += bytes;
                        gJITC.currentPage->bytesLeft -= bytes;
                }
        } while (false);
}
/*
 *      Intern.
 *      Maps ClientPage to base address
 */
static void inline jitcMapClientPage(uint32 baseaddr, ClientPage *cp)
{
        gJITC.clientPages[baseaddr >> 12] = cp;
        cp->baseaddress = baseaddr;
}

/*
 *      Intern.
 *      Unmaps ClientPage at base address
 */
static void inline jitcUnmapClientPage(uint32 baseaddr)
{
        gJITC.clientPages[baseaddr >> 12] = NULL;
}

/*
 *      Intern.
 *      Unmaps ClientPage
 */
static void inline jitcUnmapClientPage(ClientPage *cp)
{
        jitcUnmapClientPage(cp->baseaddress);
}

/*
 *      Moves client page to the end of the LRU list
 *      page *must* be in LRU list before
 */
extern "C" FASTCALL ClientPage *jitcTouchClientPage(ClientPage *cp)
{
        if (cp->moreRU) {
                // there's a page which is used more recently
                if (cp->lessRU) {
                        // we've got at least 3 pages and 
                        // cp is neither LRU nor MRU 
                        cp->moreRU->lessRU = cp->lessRU;
                        cp->lessRU->moreRU = cp->moreRU;                
                } else {
                        // page is LRU
                        gJITC.LRUpage = cp->moreRU;
                        gJITC.LRUpage->lessRU = NULL;
                }
                cp->moreRU = NULL;
                cp->lessRU = gJITC.MRUpage;
                gJITC.MRUpage->moreRU = cp;
                gJITC.MRUpage = cp;
        }
        return cp;
}

/*
 *      Puts fragments into the freeFragmentsList
 */
//#include <valgrind/valgrind.h>
void FASTCALL jitcDestroyFragments(TranslationCacheFragment *tcf)
{
        while (tcf) {
//              VALGRIND_DISCARD_TRANSLATIONS(tcf->base, FRAGMENT_SIZE);
                // FIXME: this could be done in O(1) with an additional
                // variable in ClientPage
                TranslationCacheFragment *next = tcf->prev;
                tcf->prev = gJITC.freeFragmentsList;
                gJITC.freeFragmentsList = tcf;
                tcf = next;
        }
}

/*
 *      Unmaps ClientPage and destroys fragments
 */
static void FASTCALL jitcDestroyClientPage(ClientPage *cp)
{
        // assert(cp->tcf_current)
        jitcDestroyFragments(cp->tcf_current);
        memset(cp->entrypoints, 0, sizeof cp->entrypoints);
        cp->tcf_current = NULL;
        jitcUnmapClientPage(cp);
}

/*
 *      Moves client page into the freeClientPages list
 *      (and out of the LRU list)
 *      page *must* be in LRU list before
 */
static void FASTCALL jitcFreeClientPage(ClientPage *cp)
{
        // assert(gJITC.LRUpage)
        // assert(gJITC.MRUpage)
        
        // delete page from LRU list
        if (!cp->lessRU) {
                // cp is LRU
                if (!cp->moreRU) {
                        // cp is also MRU
                        gJITC.LRUpage = gJITC.MRUpage = NULL;
                } else {
                        // assert(cp->moreRU)
                        gJITC.LRUpage = cp->moreRU;
                        gJITC.LRUpage->lessRU = NULL;
                }
        } else {
                if (!cp->moreRU) {
                        // cp is MRU
                        // assert(cp->LRUprev)
                        gJITC.MRUpage = cp->lessRU;
                        gJITC.MRUpage->moreRU = NULL;
                } else {
                        cp->moreRU->lessRU = cp->lessRU;
                        cp->lessRU->moreRU = cp->moreRU;
                }
        }
        // and move it into the freeClientPages list
        cp->moreRU = gJITC.freeClientPages;
        gJITC.freeClientPages = cp;
}

/*
 *      Destroys and frees ClientPage
 */
extern "C" void FASTCALL jitcDestroyAndFreeClientPage(ClientPage *cp)
{
        gJITC.destroy_write++;
        jitcDestroyClientPage(cp);
        jitcFreeClientPage(cp);
}

/*
 *      Destroys and touches ClientPage
 */
static void FASTCALL jitcDestroyAndTouchClientPage(ClientPage *cp)
{
        gJITC.destroy_oopages++;
        jitcDestroyClientPage(cp);
        jitcTouchClientPage(cp);
}

/*
 *      Removes and returns fragment from top of freeFragmentsList
 */
static TranslationCacheFragment *jitcGetFragment()
{
        TranslationCacheFragment *tcf = gJITC.freeFragmentsList;
        gJITC.freeFragmentsList = tcf->prev;
        tcf->prev = NULL;
        return tcf;
}

/*
 *      Returns free fragment
 *      May destroy a page to make new free fragments
 */
static TranslationCacheFragment *jitcAllocFragment()
{
        if (!gJITC.freeFragmentsList) {
                /*
                 *      There are no free fragments
                 *      -> must free a ClientPage
                 */
                gJITC.destroy_write--;  // destroy and free will increase this
                gJITC.destroy_ootc++;
                jitcDestroyAndFreeClientPage(gJITC.LRUpage);
        }
        return jitcGetFragment();
}

/*
 *      Moves page from freeClientPages at the end of the LRU list if there's
 *      a free page or destroys the LRU page and touches it
 */
extern "C" ClientPage FASTCALL *jitcCreateClientPage(uint32 baseaddr)
{
        ClientPage *cp;
        if (gJITC.freeClientPages) {
                // get page
                cp = gJITC.freeClientPages;
                gJITC.freeClientPages = gJITC.freeClientPages->moreRU;
                // and move to the end of LRU list
                if (gJITC.MRUpage) {
                        gJITC.MRUpage->moreRU = cp;
                        cp->lessRU = gJITC.MRUpage;
                        gJITC.MRUpage = cp;
                } else {
                        cp->lessRU = NULL;
                        gJITC.LRUpage = gJITC.MRUpage = cp;
                }
                cp->moreRU = NULL;
        } else {
                cp = gJITC.LRUpage;
                jitcDestroyAndTouchClientPage(cp);
                // destroy some more
                if (gJITC.LRUpage) jitcDestroyAndTouchClientPage(gJITC.LRUpage);
                if (gJITC.LRUpage) jitcDestroyAndTouchClientPage(gJITC.LRUpage);
                if (gJITC.LRUpage) jitcDestroyAndTouchClientPage(gJITC.LRUpage);
                if (gJITC.LRUpage) jitcDestroyAndTouchClientPage(gJITC.LRUpage);
        }
        jitcMapClientPage(baseaddr, cp);
        return cp;
}

/*
 *      Returns the ClientPage which maps to baseaddr or
 *      creates a new page that maps to baseaddr
 */
ClientPage FASTCALL *jitcGetOrCreateClientPage(uint32 baseaddr)
{
        ClientPage *cp = gJITC.clientPages[baseaddr >> 12];
        if (cp) {
                return cp;
        } else {
                return jitcCreateClientPage(baseaddr);
        }
}

static inline void jitcCreateEntrypoint(ClientPage *cp, uint32 ofs)
{
        cp->entrypoints[ofs >> 2] = cp->tcp;
}

static inline NativeAddress jitcGetEntrypoint(ClientPage *cp, uint32 ofs)
{
        return cp->entrypoints[ofs >> 2];
}

extern uint64 gJITCCompileTicks;
extern uint64 gJITCRunTicks;
extern uint64 gJITCRunTicksStart;

extern "C" NativeAddress FASTCALL jitcNewEntrypoint(ClientPage *cp, uint32 baseaddr, uint32 ofs)
{
/*
        gJITCRunTicks += jitcDebugGetTicks() - gJITCRunTicksStart;
        uint64 jitcCompileStartTicks = jitcDebugGetTicks();
*/
        jitcDebugLogAdd("=== jitcNewEntrypoint: %08x Beginning jitc ===\n", baseaddr+ofs);
        gJITC.currentPage = cp;
        
        jitcEmitAlign(gJITC.hostCPUCaps.loop_align);
        
        NativeAddress entry = cp->tcp;
        jitcCreateEntrypoint(cp, ofs);

        byte *physpage;
        ppc_direct_physical_memory_handle(baseaddr, physpage);

        gJITC.pc = ofs;
        jitcInvalidateAll();
        gJITC.checkedPriviledge = false;
        gJITC.checkedFloat = false;
        gJITC.checkedVector = false;
        
        // now we've setup gJITC and can start the real compilation

        while (1) {
                gJITC.current_opc = ppc_word_from_BE(*(uint32 *)(&physpage[ofs]));
                jitcDebugLogNewInstruction();
                JITCFlow flow = ppc_gen_opc();
                if (flow == flowContinue) {
                        /* nothing to do */
                } else if (flow == flowEndBlock) {
                        jitcClobberAll();

                        gJITC.checkedPriviledge = false;
                        gJITC.checkedFloat = false;
                        gJITC.checkedVector = false;
                        if (ofs+4 < 4096) {
                                jitcCreateEntrypoint(cp, ofs+4);
                        }
                } else {
                        /* flowEndBlockUnreachable */
                        break;
                }
                ofs += 4;
                if (ofs == 4096) {
                        /*
                         *      End of page.
                         *      We must use jump to the next page via 
                         *      ppc_new_pc_asm
                         */
                        jitcClobberAll();
                        asmALURegImm(X86_MOV, EAX, 4096);
                        asmJMP((NativeAddress)ppc_new_pc_rel_asm);
                        break;
                }
                gJITC.pc += 4;
        }
/*
        gJITCRunTicksStart = jitcDebugGetTicks();
        gJITCCompileTicks += jitcDebugGetTicks() - jitcCompileStartTicks;       
*/
        return entry;
}

extern "C" NativeAddress FASTCALL jitcStartTranslation(ClientPage *cp, uint32 baseaddr, uint32 ofs)
{
        cp->tcf_current = jitcAllocFragment();
        cp->tcp = cp->tcf_current->base;
        cp->bytesLeft = FRAGMENT_SIZE;
        
        return jitcNewEntrypoint(cp, baseaddr, ofs);
}

/*
 *      Called whenever the client PC changes (to a new BB)
 *      Note that entry is a physical address
 */
extern "C" NativeAddress FASTCALL jitcNewPC(uint32 entry)
{
        if (entry > gMemorySize) {
                ht_printf("entry not physical: %08x\n", entry);
                exit(-1);
        }
        uint32 baseaddr = entry & 0xfffff000;
        ClientPage *cp = jitcGetOrCreateClientPage(baseaddr);
        jitcTouchClientPage(cp);
        if (!cp->tcf_current) {
                return jitcStartTranslation(cp, baseaddr, entry & 0xfff);
        } else {
                NativeAddress ofs = jitcGetEntrypoint(cp, entry & 0xfff);
                if (ofs) {
                        return ofs;
                } else {
                        return jitcNewEntrypoint(cp, baseaddr, entry & 0xfff);
                }
        }
}

extern "C" void FASTCALL jitc_error_msr_unsupported_bits(uint32 a)
{
        ht_printf("JITC msr Error: %08x\n", a);
        exit(1);
}

extern "C" void FASTCALL jitc_error(const char *error)
{
        ht_printf("JITC Error: %s\n", error);
        exit(1);
}

extern "C" void FASTCALL jitc_error_program(uint32 a, uint32 b)
{
        if (a != 0x00020000) {  // Filter out trap exceptions, no need to report them
                ht_printf("JITC Warning: program exception: %08x %08x\n", a, b);
        }
}

extern uint8 jitcFlagsMapping[257];
extern uint8 jitcFlagsMapping2[256];
extern uint8 jitcFlagsMappingCMP_U[257];
extern uint8 jitcFlagsMappingCMP_L[257];

bool jitc_init(int maxClientPages, uint32 tcSize)
{
        memset(&gJITC, 0, sizeof gJITC);

        x86GetCaps(gJITC.hostCPUCaps);

        gJITC.translationCache = (byte*)sys_alloc_read_write_execute(tcSize);
        if (!gJITC.translationCache) return false;
        int maxPages = gMemorySize / 4096;
        gJITC.clientPages = (ClientPage **)malloc(maxPages * sizeof (ClientPage *));
        memset(gJITC.clientPages, 0, maxPages * sizeof (ClientPage *));

        // allocate fragments
        TranslationCacheFragment *tcf = (TranslationCacheFragment *)malloc(sizeof (TranslationCacheFragment));
        gJITC.freeFragmentsList = tcf;
        tcf->base = gJITC.translationCache;
        for (uint32 addr=FRAGMENT_SIZE; addr < tcSize; addr += FRAGMENT_SIZE) {
                tcf->prev = (TranslationCacheFragment *)malloc(sizeof (TranslationCacheFragment));
                tcf = tcf->prev;
                tcf->base = gJITC.translationCache + addr;
        }
        tcf->prev = NULL;
        
        // allocate client pages
        ClientPage *cp = (ClientPage *)malloc(sizeof (ClientPage));
        memset(cp->entrypoints, 0, sizeof cp->entrypoints);
        cp->tcf_current = NULL; // not translated yet
        cp->lessRU = NULL;
        gJITC.LRUpage = NULL;
        gJITC.freeClientPages = cp;
        for (int i=1; i < maxClientPages; i++) {
                cp->moreRU = (ClientPage *)malloc(sizeof (ClientPage));
                cp->moreRU->lessRU = cp;
                cp = cp->moreRU;
                
                memset(cp->entrypoints, 0, sizeof cp->entrypoints);
                cp->tcf_current = NULL; // not translated yet
        }
        cp->moreRU = NULL;
        gJITC.MRUpage = NULL;
        
        // initialize native registers
        NativeRegType *nr = (NativeRegType *)malloc(sizeof (NativeRegType));
        nr->reg = EAX;
        nr->lessRU = NULL;
        gJITC.LRUreg = nr;
        gJITC.nativeRegsList[EAX] = nr;
        for (NativeReg reg = ECX; reg <= EDI; reg=(NativeReg)(reg+1)) {
                if (reg != ESP) {
                        nr->moreRU = (NativeRegType *)malloc(sizeof (NativeRegType));
                        nr->moreRU->lessRU = nr;
                        nr = nr->moreRU;
                        nr->reg = reg;
                        gJITC.nativeRegsList[reg] = nr;
                }
        }
        nr->moreRU = NULL;
        gJITC.MRUreg = nr;

        for (int i=1; i<9; i++) {
                gJITC.floatRegPerm[i] = i;
                gJITC.floatRegPermInverse[i] = i;
        }

        jitcFlagsMapping[0] = 1<<6; // GT
        jitcFlagsMapping[1] = 1<<7; // LT
        jitcFlagsMapping[1<<8] = 1<<5; // EQ
        jitcFlagsMappingCMP_U[0] = 1<<5; // EQ
        jitcFlagsMappingCMP_U[1] = 1<<6; // GT
        jitcFlagsMappingCMP_U[1<<8] = 1<<7; // LT
        jitcFlagsMappingCMP_L[0] = 1<<1; // EQ
        jitcFlagsMappingCMP_L[1] = 1<<2; // GT
        jitcFlagsMappingCMP_L[1<<8] = 1<<3; // LT
        for (int i=0; i<256; i++) {
                switch (i & 0xc0) {
                case 0x00: // neither zero nor sign
                        jitcFlagsMapping2[i] = 1<<6; // GT
                        break;
                case 0x40: // zero flag
                        jitcFlagsMapping2[i] = 1<<5; // EQ
                        break;
                case 0x80: // sign flag
                        jitcFlagsMapping2[i] = 1<<7; // LT
                        break;
                case 0xc0: // impossible
                        jitcFlagsMapping2[i] = 0; 
                        break;
                }
        }

        for (int i=XMM0; i<=XMM_SENTINEL; i++) {
                gJITC.LRUvregs[i] = (NativeVectorReg)(i-1);
                gJITC.MRUvregs[i] = (NativeVectorReg)(i+1);
        }

        gJITC.LRUvregs[XMM0] = XMM_SENTINEL;
        gJITC.MRUvregs[XMM_SENTINEL] = XMM0;

        gJITC.nativeVectorReg = VECTREG_NO;

        /*
         *      Note that REG_NO=-1 and PPC_REG_NO=0 so this works 
         *      by accident.
         */
        memset(gJITC.clientReg, REG_NO, sizeof gJITC.clientReg);        
        memset(gJITC.nativeReg, PPC_REG_NO, sizeof gJITC.nativeReg);
        memset(gJITC.nativeRegState, rsUnused, sizeof gJITC.nativeRegState);

        memset(gJITC.n2cVectorReg, PPC_REG_NO, sizeof gJITC.n2cVectorReg);
        memset(gJITC.c2nVectorReg, VECTREG_NO, sizeof gJITC.c2nVectorReg);
        memset(gJITC.nativeVectorRegState, rsUnused, sizeof gJITC.nativeVectorRegState);

        /*
         *      This -1 register is to be read-only, and only used when
         *              needed, and must ALWAYS stay this way!
         *
         *      It's absolutely fundamental to doing NOT's with SSE
         */
        memset(&gCPU.vr[JITC_VECTOR_NEG1], 0xff, sizeof gCPU.vr[0]);

        memset(gJITC.tlb_code_eff, 0xff, sizeof gJITC.tlb_code_eff);
        memset(gJITC.tlb_data_read_eff, 0xff, sizeof gJITC.tlb_data_read_eff);
        memset(gJITC.tlb_data_write_eff, 0xff, sizeof gJITC.tlb_data_write_eff);
        gJITC.tlb_code_hits = 0;
        gJITC.tlb_data_read_hits = 0;
        gJITC.tlb_data_write_hits = 0;
        gJITC.tlb_code_misses = 0;
        gJITC.tlb_data_read_misses = 0;
        gJITC.tlb_data_write_misses = 0;
        return true;
}

void jitc_done()
{
        if (gJITC.translationCache) sys_free_read_write_execute(gJITC.translationCache);
}
1	dpavlin	1	/*
2			* PearPC
3			* jitc.cc
4			*
5			* Copyright (C) 2004 Sebastian Biallas (sb@biallas.net)
6			*
7			* This program is free software; you can redistribute it and/or modify
8			* it under the terms of the GNU General Public License version 2 as
9			* published by the Free Software Foundation.
10			*
11			* This program is distributed in the hope that it will be useful,
12			* but WITHOUT ANY WARRANTY; without even the implied warranty of
13			* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14			* GNU General Public License for more details.
15			*
16			* You should have received a copy of the GNU General Public License
17			* along with this program; if not, write to the Free Software
18			* Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
19			*/
20
21			#include <cstdlib>
22			#include <cstring>
23
24			#include "system/sys.h"
25			#include "tools/snprintf.h"
26
27			#include "jitc.h"
28			#include "jitc_debug.h"
29			#include "jitc_asm.h"
30
31			#include "ppc_dec.h"
32			#include "ppc_mmu.h"
33			#include "ppc_tools.h"
34
35			JITC gJITC;
36
37			static TranslationCacheFragment *jitcAllocFragment();
38
39			/*
40			* Intern
41			* Called whenever a new fragment is needed
42			* returns true if a new fragment was really necessary
43			*/
44			static bool FASTCALL jitcEmitNextFragment()
45			{
46			// save old
47			TranslationCacheFragment *tcf_old = gJITC.currentPage->tcf_current;
48			NativeAddress tcp_old = gJITC.currentPage->tcp;
49			// alloc new
50			gJITC.currentPage->tcf_current = jitcAllocFragment();
51			gJITC.currentPage->tcf_current->prev = tcf_old;
52			if (((uint)(gJITC.currentPage->tcf_current->base - gJITC.currentPage->tcp)) < 20) {
53			// next Fragment directly follows
54			gJITC.currentPage->bytesLeft += FRAGMENT_SIZE;
55			return false;
56			} else {
57			gJITC.currentPage->tcp = gJITC.currentPage->tcf_current->base;
58			gJITC.currentPage->bytesLeft = FRAGMENT_SIZE;
59			// hardcoded JMP from old to new fragment
60			// FIXME: use 0xeb if possible
61			tcp_old[0] = 0xe9;
62			((uint32 )&tcp_old[1]) = gJITC.currentPage->tcp - (tcp_old+5);
63			return true;
64			}
65			}
66
67			/*
68			* emit one byte of native code
69			*/
70			void FASTCALL jitcEmit1(byte b)
71			{
72			jitcDebugLogEmit(&b, 1);
73			/*
74			* We always have to leave at least 5 bytes in the fragment
75			* to issue a final JMP
76			*/
77			if (gJITC.currentPage->bytesLeft <= 5) {
78			jitcEmitNextFragment();
79			}
80			*(gJITC.currentPage->tcp++) = b;
81			gJITC.currentPage->bytesLeft--;
82			}
83
84			/*
85			* emit native code
86			*/
87			void FASTCALL jitcEmit(byte *instr, int size)
88			{
89			jitcDebugLogEmit(instr, size);
90			if ((gJITC.currentPage->bytesLeft - size) < 5) {
91			jitcEmitNextFragment();
92			}
93			memcpy(gJITC.currentPage->tcp, instr, size);
94			gJITC.currentPage->tcp += size;
95			gJITC.currentPage->bytesLeft -= size;
96			}
97
98			/*
99			* Assures that the next instruction will be
100			* emitted in the current fragment
101			*/
102			bool FASTCALL jitcEmitAssure(int size)
103			{
104			if ((gJITC.currentPage->bytesLeft - size) < 5) {
105			jitcEmitNextFragment();
106			return false;
107			}
108			return true;
109			}
110
111			void FASTCALL jitcEmitAlign(int align)
112			{
113			do {
114			int missalign = ((uint)gJITC.currentPage->tcp) % align;
115			if (missalign) {
116			int bytes = align - missalign;
117			if ((gJITC.currentPage->bytesLeft - bytes) < 5) {
118			if (jitcEmitNextFragment()) continue;
119			}
120			gJITC.currentPage->tcp += bytes;
121			gJITC.currentPage->bytesLeft -= bytes;
122			}
123			} while (false);
124			}
125			/*
126			* Intern.
127			* Maps ClientPage to base address
128			*/
129			static void inline jitcMapClientPage(uint32 baseaddr, ClientPage *cp)
130			{
131			gJITC.clientPages[baseaddr >> 12] = cp;
132			cp->baseaddress = baseaddr;
133			}
134
135			/*
136			* Intern.
137			* Unmaps ClientPage at base address
138			*/
139			static void inline jitcUnmapClientPage(uint32 baseaddr)
140			{
141			gJITC.clientPages[baseaddr >> 12] = NULL;
142			}
143
144			/*
145			* Intern.
146			* Unmaps ClientPage
147			*/
148			static void inline jitcUnmapClientPage(ClientPage *cp)
149			{
150			jitcUnmapClientPage(cp->baseaddress);
151			}
152
153			/*
154			* Moves client page to the end of the LRU list
155			* page must be in LRU list before
156			*/
157			extern "C" FASTCALL ClientPage jitcTouchClientPage(ClientPage cp)
158			{
159			if (cp->moreRU) {
160			// there's a page which is used more recently
161			if (cp->lessRU) {
162			// we've got at least 3 pages and
163			// cp is neither LRU nor MRU
164			cp->moreRU->lessRU = cp->lessRU;
165			cp->lessRU->moreRU = cp->moreRU;
166			} else {
167			// page is LRU
168			gJITC.LRUpage = cp->moreRU;
169			gJITC.LRUpage->lessRU = NULL;
170			}
171			cp->moreRU = NULL;
172			cp->lessRU = gJITC.MRUpage;
173			gJITC.MRUpage->moreRU = cp;
174			gJITC.MRUpage = cp;
175			}
176			return cp;
177			}
178
179			/*
180			* Puts fragments into the freeFragmentsList
181			*/
182			//#include <valgrind/valgrind.h>
183			void FASTCALL jitcDestroyFragments(TranslationCacheFragment *tcf)
184			{
185			while (tcf) {
186			// VALGRIND_DISCARD_TRANSLATIONS(tcf->base, FRAGMENT_SIZE);
187			// FIXME: this could be done in O(1) with an additional
188			// variable in ClientPage
189			TranslationCacheFragment *next = tcf->prev;
190			tcf->prev = gJITC.freeFragmentsList;
191			gJITC.freeFragmentsList = tcf;
192			tcf = next;
193			}
194			}
195
196			/*
197			* Unmaps ClientPage and destroys fragments
198			*/
199			static void FASTCALL jitcDestroyClientPage(ClientPage *cp)
200			{
201			// assert(cp->tcf_current)
202			jitcDestroyFragments(cp->tcf_current);
203			memset(cp->entrypoints, 0, sizeof cp->entrypoints);
204			cp->tcf_current = NULL;
205			jitcUnmapClientPage(cp);
206			}
207
208			/*
209			* Moves client page into the freeClientPages list
210			* (and out of the LRU list)
211			* page must be in LRU list before
212			*/
213			static void FASTCALL jitcFreeClientPage(ClientPage *cp)
214			{
215			// assert(gJITC.LRUpage)
216			// assert(gJITC.MRUpage)
217
218			// delete page from LRU list
219			if (!cp->lessRU) {
220			// cp is LRU
221			if (!cp->moreRU) {
222			// cp is also MRU
223			gJITC.LRUpage = gJITC.MRUpage = NULL;
224			} else {
225			// assert(cp->moreRU)
226			gJITC.LRUpage = cp->moreRU;
227			gJITC.LRUpage->lessRU = NULL;
228			}
229			} else {
230			if (!cp->moreRU) {
231			// cp is MRU
232			// assert(cp->LRUprev)
233			gJITC.MRUpage = cp->lessRU;
234			gJITC.MRUpage->moreRU = NULL;
235			} else {
236			cp->moreRU->lessRU = cp->lessRU;
237			cp->lessRU->moreRU = cp->moreRU;
238			}
239			}
240			// and move it into the freeClientPages list
241			cp->moreRU = gJITC.freeClientPages;
242			gJITC.freeClientPages = cp;
243			}
244
245			/*
246			* Destroys and frees ClientPage
247			*/
248			extern "C" void FASTCALL jitcDestroyAndFreeClientPage(ClientPage *cp)
249			{
250			gJITC.destroy_write++;
251			jitcDestroyClientPage(cp);
252			jitcFreeClientPage(cp);
253			}
254
255			/*
256			* Destroys and touches ClientPage
257			*/
258			static void FASTCALL jitcDestroyAndTouchClientPage(ClientPage *cp)
259			{
260			gJITC.destroy_oopages++;
261			jitcDestroyClientPage(cp);
262			jitcTouchClientPage(cp);
263			}
264
265			/*
266			* Removes and returns fragment from top of freeFragmentsList
267			*/
268			static TranslationCacheFragment *jitcGetFragment()
269			{
270			TranslationCacheFragment *tcf = gJITC.freeFragmentsList;
271			gJITC.freeFragmentsList = tcf->prev;
272			tcf->prev = NULL;
273			return tcf;
274			}
275
276			/*
277			* Returns free fragment
278			* May destroy a page to make new free fragments
279			*/
280			static TranslationCacheFragment *jitcAllocFragment()
281			{
282			if (!gJITC.freeFragmentsList) {
283			/*
284			* There are no free fragments
285			* -> must free a ClientPage
286			*/
287			gJITC.destroy_write--; // destroy and free will increase this
288			gJITC.destroy_ootc++;
289			jitcDestroyAndFreeClientPage(gJITC.LRUpage);
290			}
291			return jitcGetFragment();
292			}
293
294			/*
295			* Moves page from freeClientPages at the end of the LRU list if there's
296			* a free page or destroys the LRU page and touches it
297			*/
298			extern "C" ClientPage FASTCALL *jitcCreateClientPage(uint32 baseaddr)
299			{
300			ClientPage *cp;
301			if (gJITC.freeClientPages) {
302			// get page
303			cp = gJITC.freeClientPages;
304			gJITC.freeClientPages = gJITC.freeClientPages->moreRU;
305			// and move to the end of LRU list
306			if (gJITC.MRUpage) {
307			gJITC.MRUpage->moreRU = cp;
308			cp->lessRU = gJITC.MRUpage;
309			gJITC.MRUpage = cp;
310			} else {
311			cp->lessRU = NULL;
312			gJITC.LRUpage = gJITC.MRUpage = cp;
313			}
314			cp->moreRU = NULL;
315			} else {
316			cp = gJITC.LRUpage;
317			jitcDestroyAndTouchClientPage(cp);
318			// destroy some more
319			if (gJITC.LRUpage) jitcDestroyAndTouchClientPage(gJITC.LRUpage);
320			if (gJITC.LRUpage) jitcDestroyAndTouchClientPage(gJITC.LRUpage);
321			if (gJITC.LRUpage) jitcDestroyAndTouchClientPage(gJITC.LRUpage);
322			if (gJITC.LRUpage) jitcDestroyAndTouchClientPage(gJITC.LRUpage);
323			}
324			jitcMapClientPage(baseaddr, cp);
325			return cp;
326			}
327
328			/*
329			* Returns the ClientPage which maps to baseaddr or
330			* creates a new page that maps to baseaddr
331			*/
332			ClientPage FASTCALL *jitcGetOrCreateClientPage(uint32 baseaddr)
333			{
334			ClientPage *cp = gJITC.clientPages[baseaddr >> 12];
335			if (cp) {
336			return cp;
337			} else {
338			return jitcCreateClientPage(baseaddr);
339			}
340			}
341
342			static inline void jitcCreateEntrypoint(ClientPage *cp, uint32 ofs)
343			{
344			cp->entrypoints[ofs >> 2] = cp->tcp;
345			}
346
347			static inline NativeAddress jitcGetEntrypoint(ClientPage *cp, uint32 ofs)
348			{
349			return cp->entrypoints[ofs >> 2];
350			}
351
352			extern uint64 gJITCCompileTicks;
353			extern uint64 gJITCRunTicks;
354			extern uint64 gJITCRunTicksStart;
355
356			extern "C" NativeAddress FASTCALL jitcNewEntrypoint(ClientPage *cp, uint32 baseaddr, uint32 ofs)
357			{
358			/*
359			gJITCRunTicks += jitcDebugGetTicks() - gJITCRunTicksStart;
360			uint64 jitcCompileStartTicks = jitcDebugGetTicks();
361			*/
362			jitcDebugLogAdd("=== jitcNewEntrypoint: %08x Beginning jitc ===\n", baseaddr+ofs);
363			gJITC.currentPage = cp;
364
365			jitcEmitAlign(gJITC.hostCPUCaps.loop_align);
366
367			NativeAddress entry = cp->tcp;
368			jitcCreateEntrypoint(cp, ofs);
369
370			byte *physpage;
371			ppc_direct_physical_memory_handle(baseaddr, physpage);
372
373			gJITC.pc = ofs;
374			jitcInvalidateAll();
375			gJITC.checkedPriviledge = false;
376			gJITC.checkedFloat = false;
377			gJITC.checkedVector = false;
378
379			// now we've setup gJITC and can start the real compilation
380
381			while (1) {
382			gJITC.current_opc = ppc_word_from_BE((uint32 )(&physpage[ofs]));
383			jitcDebugLogNewInstruction();
384			JITCFlow flow = ppc_gen_opc();
385			if (flow == flowContinue) {
386			/* nothing to do */
387			} else if (flow == flowEndBlock) {
388			jitcClobberAll();
389
390			gJITC.checkedPriviledge = false;
391			gJITC.checkedFloat = false;
392			gJITC.checkedVector = false;
393			if (ofs+4 < 4096) {
394			jitcCreateEntrypoint(cp, ofs+4);
395			}
396			} else {
397			/* flowEndBlockUnreachable */
398			break;
399			}
400			ofs += 4;
401			if (ofs == 4096) {
402			/*
403			* End of page.
404			* We must use jump to the next page via
405			* ppc_new_pc_asm
406			*/
407			jitcClobberAll();
408			asmALURegImm(X86_MOV, EAX, 4096);
409			asmJMP((NativeAddress)ppc_new_pc_rel_asm);
410			break;
411			}
412			gJITC.pc += 4;
413			}
414			/*
415			gJITCRunTicksStart = jitcDebugGetTicks();
416			gJITCCompileTicks += jitcDebugGetTicks() - jitcCompileStartTicks;
417			*/
418			return entry;
419			}
420
421			extern "C" NativeAddress FASTCALL jitcStartTranslation(ClientPage *cp, uint32 baseaddr, uint32 ofs)
422			{
423			cp->tcf_current = jitcAllocFragment();
424			cp->tcp = cp->tcf_current->base;
425			cp->bytesLeft = FRAGMENT_SIZE;
426
427			return jitcNewEntrypoint(cp, baseaddr, ofs);
428			}
429
430			/*
431			* Called whenever the client PC changes (to a new BB)
432			* Note that entry is a physical address
433			*/
434			extern "C" NativeAddress FASTCALL jitcNewPC(uint32 entry)
435			{
436			if (entry > gMemorySize) {
437			ht_printf("entry not physical: %08x\n", entry);
438			exit(-1);
439			}
440			uint32 baseaddr = entry & 0xfffff000;
441			ClientPage *cp = jitcGetOrCreateClientPage(baseaddr);
442			jitcTouchClientPage(cp);
443			if (!cp->tcf_current) {
444			return jitcStartTranslation(cp, baseaddr, entry & 0xfff);
445			} else {
446			NativeAddress ofs = jitcGetEntrypoint(cp, entry & 0xfff);
447			if (ofs) {
448			return ofs;
449			} else {
450			return jitcNewEntrypoint(cp, baseaddr, entry & 0xfff);
451			}
452			}
453			}
454
455			extern "C" void FASTCALL jitc_error_msr_unsupported_bits(uint32 a)
456			{
457			ht_printf("JITC msr Error: %08x\n", a);
458			exit(1);
459			}
460
461			extern "C" void FASTCALL jitc_error(const char *error)
462			{
463			ht_printf("JITC Error: %s\n", error);
464			exit(1);
465			}
466
467			extern "C" void FASTCALL jitc_error_program(uint32 a, uint32 b)
468			{
469			if (a != 0x00020000) { // Filter out trap exceptions, no need to report them
470			ht_printf("JITC Warning: program exception: %08x %08x\n", a, b);
471			}
472			}
473
474			extern uint8 jitcFlagsMapping[257];
475			extern uint8 jitcFlagsMapping2[256];
476			extern uint8 jitcFlagsMappingCMP_U[257];
477			extern uint8 jitcFlagsMappingCMP_L[257];
478
479			bool jitc_init(int maxClientPages, uint32 tcSize)
480			{
481			memset(&gJITC, 0, sizeof gJITC);
482
483			x86GetCaps(gJITC.hostCPUCaps);
484
485			gJITC.translationCache = (byte*)sys_alloc_read_write_execute(tcSize);
486			if (!gJITC.translationCache) return false;
487			int maxPages = gMemorySize / 4096;
488			gJITC.clientPages = (ClientPage *)malloc(maxPages sizeof (ClientPage *));
489			memset(gJITC.clientPages, 0, maxPages * sizeof (ClientPage *));
490
491			// allocate fragments
492			TranslationCacheFragment tcf = (TranslationCacheFragment )malloc(sizeof (TranslationCacheFragment));
493			gJITC.freeFragmentsList = tcf;
494			tcf->base = gJITC.translationCache;
495			for (uint32 addr=FRAGMENT_SIZE; addr < tcSize; addr += FRAGMENT_SIZE) {
496			tcf->prev = (TranslationCacheFragment *)malloc(sizeof (TranslationCacheFragment));
497			tcf = tcf->prev;
498			tcf->base = gJITC.translationCache + addr;
499			}
500			tcf->prev = NULL;
501
502			// allocate client pages
503			ClientPage cp = (ClientPage )malloc(sizeof (ClientPage));
504			memset(cp->entrypoints, 0, sizeof cp->entrypoints);
505			cp->tcf_current = NULL; // not translated yet
506			cp->lessRU = NULL;
507			gJITC.LRUpage = NULL;
508			gJITC.freeClientPages = cp;
509			for (int i=1; i < maxClientPages; i++) {
510			cp->moreRU = (ClientPage *)malloc(sizeof (ClientPage));
511			cp->moreRU->lessRU = cp;
512			cp = cp->moreRU;
513
514			memset(cp->entrypoints, 0, sizeof cp->entrypoints);
515			cp->tcf_current = NULL; // not translated yet
516			}
517			cp->moreRU = NULL;
518			gJITC.MRUpage = NULL;
519
520			// initialize native registers
521			NativeRegType nr = (NativeRegType )malloc(sizeof (NativeRegType));
522			nr->reg = EAX;
523			nr->lessRU = NULL;
524			gJITC.LRUreg = nr;
525			gJITC.nativeRegsList[EAX] = nr;
526			for (NativeReg reg = ECX; reg <= EDI; reg=(NativeReg)(reg+1)) {
527			if (reg != ESP) {
528			nr->moreRU = (NativeRegType *)malloc(sizeof (NativeRegType));
529			nr->moreRU->lessRU = nr;
530			nr = nr->moreRU;
531			nr->reg = reg;
532			gJITC.nativeRegsList[reg] = nr;
533			}
534			}
535			nr->moreRU = NULL;
536			gJITC.MRUreg = nr;
537
538			for (int i=1; i<9; i++) {
539			gJITC.floatRegPerm[i] = i;
540			gJITC.floatRegPermInverse[i] = i;
541			}
542
543			jitcFlagsMapping[0] = 1<<6; // GT
544			jitcFlagsMapping[1] = 1<<7; // LT
545			jitcFlagsMapping[1<<8] = 1<<5; // EQ
546			jitcFlagsMappingCMP_U[0] = 1<<5; // EQ
547			jitcFlagsMappingCMP_U[1] = 1<<6; // GT
548			jitcFlagsMappingCMP_U[1<<8] = 1<<7; // LT
549			jitcFlagsMappingCMP_L[0] = 1<<1; // EQ
550			jitcFlagsMappingCMP_L[1] = 1<<2; // GT
551			jitcFlagsMappingCMP_L[1<<8] = 1<<3; // LT
552			for (int i=0; i<256; i++) {
553			switch (i & 0xc0) {
554			case 0x00: // neither zero nor sign
555			jitcFlagsMapping2[i] = 1<<6; // GT
556			break;
557			case 0x40: // zero flag
558			jitcFlagsMapping2[i] = 1<<5; // EQ
559			break;
560			case 0x80: // sign flag
561			jitcFlagsMapping2[i] = 1<<7; // LT
562			break;
563			case 0xc0: // impossible
564			jitcFlagsMapping2[i] = 0;
565			break;
566			}
567			}
568
569			for (int i=XMM0; i<=XMM_SENTINEL; i++) {
570			gJITC.LRUvregs[i] = (NativeVectorReg)(i-1);
571			gJITC.MRUvregs[i] = (NativeVectorReg)(i+1);
572			}
573
574			gJITC.LRUvregs[XMM0] = XMM_SENTINEL;
575			gJITC.MRUvregs[XMM_SENTINEL] = XMM0;
576
577			gJITC.nativeVectorReg = VECTREG_NO;
578
579			/*
580			* Note that REG_NO=-1 and PPC_REG_NO=0 so this works
581			* by accident.
582			*/
583			memset(gJITC.clientReg, REG_NO, sizeof gJITC.clientReg);
584			memset(gJITC.nativeReg, PPC_REG_NO, sizeof gJITC.nativeReg);
585			memset(gJITC.nativeRegState, rsUnused, sizeof gJITC.nativeRegState);
586
587			memset(gJITC.n2cVectorReg, PPC_REG_NO, sizeof gJITC.n2cVectorReg);
588			memset(gJITC.c2nVectorReg, VECTREG_NO, sizeof gJITC.c2nVectorReg);
589			memset(gJITC.nativeVectorRegState, rsUnused, sizeof gJITC.nativeVectorRegState);
590
591			/*
592			* This -1 register is to be read-only, and only used when
593			* needed, and must ALWAYS stay this way!
594			*
595			* It's absolutely fundamental to doing NOT's with SSE
596			*/
597			memset(&gCPU.vr[JITC_VECTOR_NEG1], 0xff, sizeof gCPU.vr[0]);
598
599			memset(gJITC.tlb_code_eff, 0xff, sizeof gJITC.tlb_code_eff);
600			memset(gJITC.tlb_data_read_eff, 0xff, sizeof gJITC.tlb_data_read_eff);
601			memset(gJITC.tlb_data_write_eff, 0xff, sizeof gJITC.tlb_data_write_eff);
602			gJITC.tlb_code_hits = 0;
603			gJITC.tlb_data_read_hits = 0;
604			gJITC.tlb_data_write_hits = 0;
605			gJITC.tlb_code_misses = 0;
606			gJITC.tlb_data_read_misses = 0;
607			gJITC.tlb_data_write_misses = 0;
608			return true;
609			}
610
611			void jitc_done()
612			{
613			if (gJITC.translationCache) sys_free_read_write_execute(gJITC.translationCache);
614			}