+
+ if (_cpu_capabilities & k64Bit) // 64-bit processors use SPRG3 for TLS
+ _cpu_capabilities |= kFastThreadLocalStorage;
+}
+
+
+/* Copy data into commpage. */
+
+static void
+commpage_stuff(
+ int address,
+ const void *source,
+ int length )
+{
+ char *dest = commpage_addr_of(address);
+
+ if (dest < next)
+ panic("commpage overlap: %p - %p", dest, next);
+
+ bcopy((const char*)source,dest,length);
+
+ next = (dest + length);
+}
+
+
+/* Modify commpage code in-place for this specific platform. */
+
+static void
+commpage_change(
+ uint32_t *ptr,
+ int bytes,
+ uint32_t search_mask,
+ uint32_t search_pattern,
+ uint32_t new_mask,
+ uint32_t new_pattern,
+ int (*check)(uint32_t instruction) )
+{
+ int words = bytes >> 2;
+ uint32_t word;
+
+ while( (--words) >= 0 ) {
+ word = *ptr;
+ if ((word & search_mask)==search_pattern) {
+ if ((check==NULL) || (check(word))) { // check instruction if necessary
+ word &= ~new_mask;
+ word |= new_pattern;
+ *ptr = word;
+ }
+ }
+ ptr++;
+ }
+}
+
+
+/* Check to see if exactly one bit is set in a MTCRF instruction's FXM field.
+ */
+static int
+commpage_onebit(
+ uint32_t mtcrf )
+{
+ int x = (mtcrf >> 12) & 0xFF; // isolate the FXM field of the MTCRF
+
+ if (x==0)
+ panic("commpage bad mtcrf");
+
+ return (x & (x-1))==0 ? 1 : 0; // return 1 iff exactly 1 bit set in FXM field
+}
+
+
+/* Check to see if a RLWINM (whose ME is 31) is a SRWI. Since to shift right n bits
+ * you must "RLWINM ra,rs,32-n,n,31", if (SH+MB)==32 then we have a SRWI.
+ */
+static int
+commpage_srwi(
+ uint32_t rlwinm )
+{
+ int sh = (rlwinm >> 11) & 0x1F; // extract SH field of RLWINM, ie bits 16-20
+ int mb = (rlwinm >> 6 ) & 0x1F; // extract MB field of RLWINM, ie bits 21-25
+
+ return (sh + mb) == 32; // it is a SRWI if (SH+MB)==32
+}
+
+
+/* Handle kCommPageDCBA bit: the commpage routine uses DCBA. If the machine we're
+ * running on doesn't benefit from use of that instruction, map them to NOPs
+ * in the commpage.
+ */
+static void
+commpage_handle_dcbas(
+ int address,
+ int length )
+{
+ uint32_t *ptr, search_mask, search, replace_mask, replace;
+
+ if ( (_cpu_capabilities & kDcbaRecommended) == 0 ) {
+ ptr = commpage_addr_of(address);
+
+ search_mask = 0xFC0007FE; // search x-form opcode bits
+ search = 0x7C0005EC; // for a DCBA
+ replace_mask = 0xFFFFFFFF; // replace all bits...
+ replace = 0x60000000; // ...with a NOP
+
+ commpage_change(ptr,length,search_mask,search,replace_mask,replace,NULL);
+ }
+}
+
+
+/* Handle kCommPageSYNC bit: this routine uses SYNC, LWSYNC, or EIEIO. If we're
+ * running on a UP machine, map them to NOPs.
+ */
+static void
+commpage_handle_syncs(
+ int address,
+ int length )
+{
+ uint32_t *ptr, search_mask, search, replace_mask, replace;
+
+ if (_NumCPUs() == 1) {
+ ptr = commpage_addr_of(address);
+
+ search_mask = 0xFC0005FE; // search x-form opcode bits (but ignore bit 0x00000200)
+ search = 0x7C0004AC; // for a SYNC, LWSYNC, or EIEIO
+ replace_mask = 0xFFFFFFFF; // replace all bits...
+ replace = 0x60000000; // ...with a NOP
+
+ commpage_change(ptr,length,search_mask,search,replace_mask,replace,NULL);
+ }
+}
+
+
+/* Handle kCommPageISYNC bit: this routine uses ISYNCs. If we're running on a UP machine,
+ * map them to NOPs.
+ */
+static void
+commpage_handle_isyncs(
+ int address,
+ int length )
+{
+ uint32_t *ptr, search_mask, search, replace_mask, replace;
+
+ if (_NumCPUs() == 1) {
+ ptr = commpage_addr_of(address);
+
+ search_mask = 0xFC0007FE; // search xl-form opcode bits
+ search = 0x4C00012C; // for an ISYNC
+ replace_mask = 0xFFFFFFFF; // replace all bits...
+ replace = 0x60000000; // ...with a NOP
+
+ commpage_change(ptr,length,search_mask,search,replace_mask,replace,NULL);
+ }
+}
+
+
+/* Handle kCommPageMTCRF bit. When this was written (3/03), the assembler did not
+ * recognize the special form of MTCRF instructions, in which exactly one bit is set
+ * in the 8-bit mask field. Bit 11 of the instruction should be set in this case,
+ * since the 970 and probably other 64-bit processors optimize it. Once the assembler
+ * has been updated this code can be removed, though it need not be.
+ */
+static void
+commpage_handle_mtcrfs(
+ int address,
+ int length )
+{
+ uint32_t *ptr, search_mask, search, replace_mask, replace;
+
+ if (_cpu_capabilities & k64Bit) {
+ ptr = commpage_addr_of(address);
+
+ search_mask = 0xFC0007FE; // search x-form opcode bits
+ search = 0x7C000120; // for a MTCRF
+ replace_mask = 0x00100000; // replace bit 11...
+ replace = 0x00100000; // ...with a 1-bit
+
+ commpage_change(ptr,length,search_mask,search,replace_mask,replace,commpage_onebit);
+ }
+}
+
+
+/* Port 32-bit code to 64-bit for use in the 64-bit commpage. This sounds fancier than
+ * it is. We do the following:
+ * - map "cmpw*" into "cmpd*"
+ * - map "srwi" into "srdi"
+ * Perhaps surprisingly, this is enough to permit lots of code to run in 64-bit mode, as
+ * long as it is written with this in mind.
+ */
+static void
+commpage_port_32_to_64(
+ int address,
+ int length )
+{
+ uint32_t *ptr, search_mask, search, replace_mask, replace;
+
+ ptr = commpage_addr_of(address);
+
+ search_mask = 0xFC2007FE; // search x-form opcode bits (and L bit)
+ search = 0x7C000000; // for a CMPW
+ replace_mask = 0x00200000; // replace bit 10 (L)...
+ replace = 0x00200000; // ...with a 1-bit, converting word to doubleword compares
+ commpage_change(ptr,length,search_mask,search,replace_mask,replace,NULL);
+
+ search_mask = 0xFC2007FE; // search x-form opcode bits (and L bit)
+ search = 0x7C000040; // for a CMPLW
+ replace_mask = 0x00200000; // replace bit 10 (L)...
+ replace = 0x00200000; // ...with a 1-bit, converting word to doubleword compares
+ commpage_change(ptr,length,search_mask,search,replace_mask,replace,NULL);
+
+ search_mask = 0xFC200000; // search d-form opcode bits (and L bit)
+ search = 0x28000000; // for a CMPLWI
+ replace_mask = 0x00200000; // replace bit 10 (L)...
+ replace = 0x00200000; // ...with a 1-bit, converting word to doubleword compares
+ commpage_change(ptr,length,search_mask,search,replace_mask,replace,NULL);
+
+ search_mask = 0xFC200000; // search d-form opcode bits (and L bit)
+ search = 0x2C000000; // for a CMPWI
+ replace_mask = 0x00200000; // replace bit 10 (L)...
+ replace = 0x00200000; // ...with a 1-bit, converting word to doubleword compares
+ commpage_change(ptr,length,search_mask,search,replace_mask,replace,NULL);
+
+ search_mask = 0xFC00003E; // search d-form opcode bits and ME (mask end) field
+ search = 0x5400003E; // for an RLWINM with ME=31 (which might be a "srwi")
+ replace_mask = 0xFC00003E; // then replace RLWINM's opcode and ME field to make a RLDICL
+ replace = 0x78000002; // opcode is 30, ME is 0, except we add 32 to SH amount
+ commpage_change(ptr,length,search_mask,search,replace_mask,replace,commpage_srwi);
+}
+
+
+/* Copy a routine into comm page if it matches running machine.
+ */
+static void
+commpage_stuff_routine(
+ commpage_descriptor *rd,
+ int mode ) // kCommPage32 or kCommPage64
+{
+ char *routine_code;
+ int must,cant;
+
+ if ( (rd->special & mode) == 0 ) // is this routine useable in this mode?
+ return;
+
+ if (rd->commpage_address != cur_routine) {
+ if ((cur_routine!=0) && (matched==0))
+ panic("commpage no match for last, next address %08x", rd->commpage_address);
+ cur_routine = rd->commpage_address;
+ matched = 0;
+ }
+
+ must = _cpu_capabilities & rd->musthave;
+ cant = _cpu_capabilities & rd->canthave;
+
+ if ((must == rd->musthave) && (cant == 0)) {
+ if (matched)
+ panic("commpage multiple matches for address %08x", rd->commpage_address);
+ matched = 1;
+ routine_code = ((char*)rd) + rd->code_offset;
+
+ commpage_stuff(rd->commpage_address,routine_code,rd->code_length);
+
+ if (rd->special & kCommPageDCBA)
+ commpage_handle_dcbas(rd->commpage_address,rd->code_length);
+
+ if (rd->special & kCommPageSYNC)
+ commpage_handle_syncs(rd->commpage_address,rd->code_length);
+
+ if (rd->special & kCommPageISYNC)
+ commpage_handle_isyncs(rd->commpage_address,rd->code_length);
+
+ if (rd->special & kCommPageMTCRF)
+ commpage_handle_mtcrfs(rd->commpage_address,rd->code_length);
+
+ if ((mode == kCommPage64) && (rd->special & kPort32to64))
+ commpage_port_32_to_64(rd->commpage_address,rd->code_length);
+ }
+}
+
+
+/* Fill in the 32- or 64-bit commpage. Called once for each. */
+
+static void
+commpage_populate_one(
+ vm_map_t submap, // the map to populate
+ char ** kernAddressPtr, // address within kernel of this commpage
+ int mode, // either kCommPage32 or kCommPage64
+ const char* signature ) // "commpage 32-bit" or "commpage 64-bit"
+{
+ char c1;
+ short c2;
+ addr64_t c8;
+ static double two52 = 1048576.0 * 1048576.0 * 4096.0; // 2**52
+ static double ten6 = 1000000.0; // 10**6
+ static uint64_t magicFE = 0xFEFEFEFEFEFEFEFFLL; // used to find 0s in strings
+ static uint64_t magic80 = 0x8080808080808080LL; // also used to find 0s
+ commpage_descriptor **rd;
+ short version = _COMM_PAGE_THIS_VERSION;
+
+ next = NULL; // initialize next available byte in the commpage
+ cur_routine = 0; // initialize comm page address of "current" routine
+
+ commPagePtr = (char*) commpage_allocate( submap );
+ *kernAddressPtr = commPagePtr; // save address either in commPagePtr32 or 64
+
+ /* Stuff in the constants. We move things into the comm page in strictly
+ * ascending order, so we can check for overlap and panic if so.
+ */
+
+ commpage_stuff(_COMM_PAGE_SIGNATURE,signature,strlen(signature));
+
+ commpage_stuff(_COMM_PAGE_VERSION,&version,2);
+
+ commpage_stuff(_COMM_PAGE_CPU_CAPABILITIES,&_cpu_capabilities,sizeof(int));
+
+ c1 = (_cpu_capabilities & kHasAltivec) ? -1 : 0;
+ commpage_stuff(_COMM_PAGE_ALTIVEC,&c1,1);
+
+ c1 = (_cpu_capabilities & k64Bit) ? -1 : 0;
+ commpage_stuff(_COMM_PAGE_64_BIT,&c1,1);
+
+ if (_cpu_capabilities & kCache32)
+ c2 = 32;
+ else if (_cpu_capabilities & kCache64)
+ c2 = 64;
+ else if (_cpu_capabilities & kCache128)
+ c2 = 128;
+ commpage_stuff(_COMM_PAGE_CACHE_LINESIZE,&c2,2);
+
+ commpage_stuff(_COMM_PAGE_2_TO_52,&two52,8);
+ commpage_stuff(_COMM_PAGE_10_TO_6,&ten6,8);
+ commpage_stuff(_COMM_PAGE_MAGIC_FE,&magicFE,8);
+ commpage_stuff(_COMM_PAGE_MAGIC_80,&magic80,8);
+
+ c8 = 0; // 0 timestamp means "disabled"
+ commpage_stuff(_COMM_PAGE_TIMEBASE,&c8,8);
+ commpage_stuff(_COMM_PAGE_TIMESTAMP,&c8,8);
+ commpage_stuff(_COMM_PAGE_SEC_PER_TICK,&c8,8);
+
+ /* Now the routines. We try each potential routine in turn,
+ * and copy in any that "match" the platform we are running on.
+ * We require that exactly one routine match for each slot in the
+ * comm page, and panic if not.
+ */
+
+ for( rd = routines; *rd != NULL ; rd++ )
+ commpage_stuff_routine(*rd,mode);
+
+ if (!matched)
+ panic("commpage no match on last routine");
+
+ if (next > (commPagePtr + _COMM_PAGE_AREA_USED))
+ panic("commpage overflow");
+
+
+ // make all that new code executable
+
+ sync_cache_virtual((vm_offset_t) commPagePtr,_COMM_PAGE_AREA_USED);
projects / apple / xnu.git / blobdiff