pexpert/arm/pe_serial.c

   1 /*
   2  * Copyright (c) 2000-2015 Apple Inc. All rights reserved.
   3  */
   4
   5 /*
   6  * file: pe_serial.c Polled-mode UART0 driver for S3c2410 and PL011.
   7  */
   8
   9
  10 #include <kern/clock.h>
  11 #include <kern/debug.h>
  12 #include <libkern/OSBase.h>
  13 #include <libkern/section_keywords.h>
  14 #include <mach/mach_time.h>
  15 #include <machine/atomic.h>
  16 #include <machine/machine_routines.h>
  17 #include <pexpert/pexpert.h>
  18 #include <pexpert/protos.h>
  19 #include <pexpert/device_tree.h>
  20 #if defined __arm__
  21 #include <arm/caches_internal.h>
  22 #include <arm/machine_routines.h>
  23 #include <arm/proc_reg.h>
  24 #include <pexpert/arm/board_config.h>
  25 #include <vm/pmap.h>
  26 #elif defined __arm64__
  27 #include <pexpert/arm/consistent_debug.h>
  28 #include <pexpert/arm64/board_config.h>
  29 #include <arm64/proc_reg.h>
  30 #endif
  31 #if HIBERNATION
  32 #include <machine/pal_hibernate.h>
  33 #endif /* HIBERNATION */
  34
  35 struct pe_serial_functions {
  36         void            (*uart_init) (void);
  37         void            (*uart_set_baud_rate) (int unit, uint32_t baud_rate);
  38         int             (*tr0) (void);
  39         void            (*td0) (int c);
  40         int             (*rr0) (void);
  41         int             (*rd0) (void);
  42         struct pe_serial_functions *next;
  43 };
  44
  45 SECURITY_READ_ONLY_LATE(static struct pe_serial_functions*) gPESF = NULL;
  46
  47 static int         uart_initted = 0;    /* 1 if init'ed */
  48 static vm_offset_t uart_base = 0;
  49
  50 /*****************************************************************************/
  51
  52 #ifdef  S3CUART
  53
  54 static int32_t dt_pclk      = -1;
  55 static int32_t dt_sampling  = -1;
  56 static int32_t dt_ubrdiv    = -1;
  57
  58 static void ln2410_uart_set_baud_rate(__unused int unit, uint32_t baud_rate);
  59
  60 static void
  61 ln2410_uart_init(void)
  62 {
  63         uint32_t ucon0 = 0x405; /* NCLK, No interrupts, No DMA - just polled */
  64
  65         rULCON0 = 0x03;         /* 81N, not IR */
  66
  67         // Override with pclk dt entry
  68         if (dt_pclk != -1) {
  69                 ucon0 = ucon0 & ~0x400;
  70         }
  71
  72         rUCON0 = ucon0;
  73         rUMCON0 = 0x00;         /* Clear Flow Control */
  74
  75         ln2410_uart_set_baud_rate(0, 115200);
  76
  77         rUFCON0 = 0x03;         /* Clear & Enable FIFOs */
  78         rUMCON0 = 0x01;         /* Assert RTS on UART0 */
  79 }
  80
  81 static void
  82 ln2410_uart_set_baud_rate(__unused int unit, uint32_t baud_rate)
  83 {
  84         uint32_t div = 0;
  85         uint32_t uart_clock = 0;
  86         uint32_t sample_rate = 16;
  87
  88         if (baud_rate < 300) {
  89                 baud_rate = 9600;
  90         }
  91
  92         if (rUCON0 & 0x400) {
  93                 // NCLK
  94                 uart_clock = (uint32_t)gPEClockFrequencyInfo.fix_frequency_hz;
  95         } else {
  96                 // PCLK
  97                 uart_clock = (uint32_t)gPEClockFrequencyInfo.prf_frequency_hz;
  98         }
  99
 100         if (dt_sampling != -1) {
 101                 // Use the sampling rate specified in the Device Tree
 102                 sample_rate = dt_sampling & 0xf;
 103         }
 104
 105         if (dt_ubrdiv != -1) {
 106                 // Use the ubrdiv specified in the Device Tree
 107                 div = dt_ubrdiv & 0xffff;
 108         } else {
 109                 // Calculate ubrdiv. UBRDIV = (SourceClock / (BPS * Sample Rate)) - 1
 110                 div = uart_clock / (baud_rate * sample_rate);
 111
 112                 uint32_t actual_baud = uart_clock / ((div + 0) * sample_rate);
 113                 uint32_t baud_low    = uart_clock / ((div + 1) * sample_rate);
 114
 115                 // Adjust div to get the closest target baudrate
 116                 if ((baud_rate - baud_low) > (actual_baud - baud_rate)) {
 117                         div--;
 118                 }
 119         }
 120
 121         // Sample Rate [19:16], UBRDIV [15:0]
 122         rUBRDIV0 = ((16 - sample_rate) << 16) | div;
 123 }
 124
 125 static int
 126 ln2410_tr0(void)
 127 {
 128         return rUTRSTAT0 & 0x04;
 129 }
 130 static void
 131 ln2410_td0(int c)
 132 {
 133         rUTXH0 = (unsigned)(c & 0xff);
 134 }
 135 static int
 136 ln2410_rr0(void)
 137 {
 138         return rUTRSTAT0 & 0x01;
 139 }
 140 static int
 141 ln2410_rd0(void)
 142 {
 143         return (int)rURXH0;
 144 }
 145
 146 SECURITY_READ_ONLY_LATE(static struct pe_serial_functions) ln2410_serial_functions =
 147 {
 148         .uart_init = ln2410_uart_init,
 149         .uart_set_baud_rate = ln2410_uart_set_baud_rate,
 150         .tr0 = ln2410_tr0,
 151         .td0 = ln2410_td0,
 152         .rr0 = ln2410_rr0,
 153         .rd0 = ln2410_rd0
 154 };
 155
 156 #endif  /* S3CUART */
 157
 158 /*****************************************************************************/
 159
 160 static void
 161 dcc_uart_init(void)
 162 {
 163 }
 164
 165 static unsigned int
 166 read_dtr(void)
 167 {
 168 #ifdef __arm__
 169         unsigned int    c;
 170         __asm__ volatile (
 171                  "mrc p14, 0, %0, c0, c5\n"
 172  :               "=r"(c));
 173         return c;
 174 #else
 175         /* ARM64_TODO */
 176         panic_unimplemented();
 177         return 0;
 178 #endif
 179 }
 180 static void
 181 write_dtr(unsigned int c)
 182 {
 183 #ifdef __arm__
 184         __asm__ volatile (
 185                  "mcr p14, 0, %0, c0, c5\n"
 186                  :
 187                  :"r"(c));
 188 #else
 189         /* ARM64_TODO */
 190         (void)c;
 191         panic_unimplemented();
 192 #endif
 193 }
 194
 195 static int
 196 dcc_tr0(void)
 197 {
 198 #ifdef __arm__
 199         return !(arm_debug_read_dscr() & ARM_DBGDSCR_TXFULL);
 200 #else
 201         /* ARM64_TODO */
 202         panic_unimplemented();
 203         return 0;
 204 #endif
 205 }
 206
 207 static void
 208 dcc_td0(int c)
 209 {
 210         write_dtr(c);
 211 }
 212
 213 static int
 214 dcc_rr0(void)
 215 {
 216 #ifdef __arm__
 217         return arm_debug_read_dscr() & ARM_DBGDSCR_RXFULL;
 218 #else
 219         /* ARM64_TODO */
 220         panic_unimplemented();
 221         return 0;
 222 #endif
 223 }
 224
 225 static int
 226 dcc_rd0(void)
 227 {
 228         return read_dtr();
 229 }
 230
 231 SECURITY_READ_ONLY_LATE(static struct pe_serial_functions) dcc_serial_functions =
 232 {
 233         .uart_init = dcc_uart_init,
 234         .uart_set_baud_rate = NULL,
 235         .tr0 = dcc_tr0,
 236         .td0 = dcc_td0,
 237         .rr0 = dcc_rr0,
 238         .rd0 = dcc_rd0
 239 };
 240
 241 /*****************************************************************************/
 242
 243 #ifdef SHMCON
 244
 245 #define CPU_CACHELINE_SIZE      (1 << MMU_CLINE)
 246
 247 #ifndef SHMCON_NAME
 248 #define SHMCON_NAME             "AP-xnu"
 249 #endif
 250
 251 #define SHMCON_MAGIC            'SHMC'
 252 #define SHMCON_VERSION          2
 253 #define CBUF_IN                 0
 254 #define CBUF_OUT                1
 255 #define INBUF_SIZE              (panic_size / 16)
 256 #define FULL_ALIGNMENT          (64)
 257
 258 #define FLAG_CACHELINE_32       1
 259 #define FLAG_CACHELINE_64       2
 260
 261 /* Defines to clarify the master/slave fields' use as circular buffer pointers */
 262 #define head_in         sidx[CBUF_IN]
 263 #define tail_in         midx[CBUF_IN]
 264 #define head_out        midx[CBUF_OUT]
 265 #define tail_out        sidx[CBUF_OUT]
 266
 267 /* TODO: get from device tree/target */
 268 #define NUM_CHILDREN            5
 269
 270 #define WRAP_INCR(len, x) do{ (x)++; if((x) >= (len)) (x) = 0; } while(0)
 271 #define ROUNDUP(a, b) (((a) + ((b) - 1)) & (~((b) - 1)))
 272
 273 #define MAX(a, b) ((a) > (b) ? (a) : (b))
 274 #define MIN(a, b) ((a) < (b) ? (a) : (b))
 275
 276 #define shmcon_barrier() do {__asm__ volatile("dmb ish" : : : "memory");} while(0)
 277
 278 struct shm_buffer_info {
 279         uint64_t        base;
 280         uint32_t        unused;
 281         uint32_t        magic;
 282 };
 283
 284 struct shmcon_header {
 285         uint32_t        magic;
 286         uint8_t         version;
 287         uint8_t         children;       /* number of child entries in child_ent */
 288         uint16_t        flags;
 289         uint64_t        buf_paddr[2];   /* Physical address for buffers (in, out) */
 290         uint32_t        buf_len[2];
 291         uint8_t         name[8];
 292
 293         /* Slave-modified data - invalidate before read */
 294         uint32_t        sidx[2] __attribute__((aligned(FULL_ALIGNMENT)));       /* In head, out tail */
 295
 296         /* Master-modified data - clean after write */
 297         uint32_t        midx[2] __attribute__((aligned(FULL_ALIGNMENT)));       /* In tail, out head */
 298
 299         uint64_t        child[0];       /* Physical address of child header pointers */
 300 };
 301
 302 static volatile struct shmcon_header *shmcon = NULL;
 303 static volatile uint8_t *shmbuf[2];
 304 #ifdef SHMCON_THROTTLED
 305 static uint64_t grace = 0;
 306 static uint64_t full_timeout = 0;
 307 #endif
 308
 309 static void
 310 shmcon_set_baud_rate(__unused int unit, __unused uint32_t baud_rate)
 311 {
 312         return;
 313 }
 314
 315 static int
 316 shmcon_tr0(void)
 317 {
 318 #ifdef SHMCON_THROTTLED
 319         uint32_t head = shmcon->head_out;
 320         uint32_t tail = shmcon->tail_out;
 321         uint32_t len = shmcon->buf_len[CBUF_OUT];
 322
 323         WRAP_INCR(len, head);
 324         if (head != tail) {
 325                 full_timeout = 0;
 326                 return 1;
 327         }
 328
 329         /* Full.  Is this buffer being serviced? */
 330         if (full_timeout == 0) {
 331                 full_timeout = mach_absolute_time() + grace;
 332                 return 0;
 333         }
 334         if (full_timeout > mach_absolute_time()) {
 335                 return 0;
 336         }
 337
 338         /* Timeout - slave not really there or not keeping up */
 339         tail += (len / 4);
 340         if (tail >= len) {
 341                 tail -= len;
 342         }
 343         shmcon_barrier();
 344         shmcon->tail_out = tail;
 345         full_timeout = 0;
 346 #endif
 347         return 1;
 348 }
 349
 350 static void
 351 shmcon_td0(int c)
 352 {
 353         uint32_t head = shmcon->head_out;
 354         uint32_t len = shmcon->buf_len[CBUF_OUT];
 355
 356         shmbuf[CBUF_OUT][head] = (uint8_t)c;
 357         WRAP_INCR(len, head);
 358         shmcon_barrier();
 359         shmcon->head_out = head;
 360 }
 361
 362 static int
 363 shmcon_rr0(void)
 364 {
 365         if (shmcon->tail_in == shmcon->head_in) {
 366                 return 0;
 367         }
 368         return 1;
 369 }
 370
 371 static int
 372 shmcon_rd0(void)
 373 {
 374         int c;
 375         uint32_t tail = shmcon->tail_in;
 376         uint32_t len = shmcon->buf_len[CBUF_IN];
 377
 378         c = shmbuf[CBUF_IN][tail];
 379         WRAP_INCR(len, tail);
 380         shmcon_barrier();
 381         shmcon->tail_in = tail;
 382         return c;
 383 }
 384
 385 static void
 386 shmcon_init(void)
 387 {
 388         DTEntry                         entry;
 389         uintptr_t const                 *reg_prop;
 390         volatile struct shm_buffer_info *end;
 391         size_t                          i, header_size;
 392         unsigned int                    size;
 393         vm_offset_t                     pa_panic_base, panic_size, va_buffer_base, va_buffer_end;
 394
 395         if (kSuccess != SecureDTLookupEntry(0, "pram", &entry)) {
 396                 return;
 397         }
 398
 399         if (kSuccess != SecureDTGetProperty(entry, "reg", (void const **)&reg_prop, &size)) {
 400                 return;
 401         }
 402
 403         pa_panic_base = reg_prop[0];
 404         panic_size = reg_prop[1];
 405
 406         shmcon = (struct shmcon_header *)ml_map_high_window(pa_panic_base, panic_size);
 407         header_size = sizeof(*shmcon) + (NUM_CHILDREN * sizeof(shmcon->child[0]));
 408         va_buffer_base = ROUNDUP((uintptr_t)(shmcon) + header_size, CPU_CACHELINE_SIZE);
 409         va_buffer_end  = (uintptr_t)shmcon + panic_size - (sizeof(*end));
 410
 411         if ((shmcon->magic == SHMCON_MAGIC) && (shmcon->version == SHMCON_VERSION)) {
 412                 vm_offset_t pa_buffer_base, pa_buffer_end;
 413
 414                 pa_buffer_base = ml_vtophys(va_buffer_base);
 415                 pa_buffer_end  = ml_vtophys(va_buffer_end);
 416
 417                 /* Resume previous console session */
 418                 for (i = 0; i < 2; i++) {
 419                         vm_offset_t pa_buf;
 420                         uint32_t len;
 421
 422                         pa_buf = (uintptr_t)shmcon->buf_paddr[i];
 423                         len = shmcon->buf_len[i];
 424                         /* Validate buffers */
 425                         if ((pa_buf < pa_buffer_base) ||
 426                             (pa_buf >= pa_buffer_end) ||
 427                             ((pa_buf + len) > pa_buffer_end) ||
 428                             (shmcon->midx[i] >= len) ||     /* Index out of bounds */
 429                             (shmcon->sidx[i] >= len) ||
 430                             (pa_buf != ROUNDUP(pa_buf, CPU_CACHELINE_SIZE)) ||     /* Unaligned pa_buffer */
 431                             (len < 1024) ||
 432                             (len > (pa_buffer_end - pa_buffer_base)) ||
 433                             (shmcon->children != NUM_CHILDREN)) {
 434                                 goto validation_failure;
 435                         }
 436                         /* Compute the VA offset of the buffer */
 437                         shmbuf[i] = (uint8_t *)(uintptr_t)shmcon + ((uintptr_t)pa_buf - (uintptr_t)pa_panic_base);
 438                 }
 439                 /* Check that buffers don't overlap */
 440                 if ((uintptr_t)shmbuf[0] < (uintptr_t)shmbuf[1]) {
 441                         if ((uintptr_t)(shmbuf[0] + shmcon->buf_len[0]) > (uintptr_t)shmbuf[1]) {
 442                                 goto validation_failure;
 443                         }
 444                 } else {
 445                         if ((uintptr_t)(shmbuf[1] + shmcon->buf_len[1]) > (uintptr_t)shmbuf[0]) {
 446                                 goto validation_failure;
 447                         }
 448                 }
 449                 shmcon->tail_in = shmcon->head_in; /* Clear input buffer */
 450                 shmcon_barrier();
 451         } else {
 452 validation_failure:
 453                 shmcon->magic = 0;
 454                 shmcon_barrier();
 455                 shmcon->buf_len[CBUF_IN] = (uint32_t)INBUF_SIZE;
 456                 shmbuf[CBUF_IN]  = (uint8_t *)va_buffer_base;
 457                 shmbuf[CBUF_OUT] = (uint8_t *)ROUNDUP(va_buffer_base + INBUF_SIZE, CPU_CACHELINE_SIZE);
 458                 for (i = 0; i < 2; i++) {
 459                         shmcon->midx[i] = 0;
 460                         shmcon->sidx[i] = 0;
 461                         shmcon->buf_paddr[i] = (uintptr_t)ml_vtophys((vm_offset_t)shmbuf[i]);
 462                 }
 463                 shmcon->buf_len[CBUF_OUT] = (uint32_t)(va_buffer_end - (uintptr_t)shmbuf[CBUF_OUT]);
 464                 shmcon->version = SHMCON_VERSION;
 465 #pragma clang diagnostic push
 466 #pragma clang diagnostic ignored "-Wcast-qual"
 467                 memset((void *)shmcon->name, ' ', sizeof(shmcon->name));
 468                 memcpy((void *)shmcon->name, SHMCON_NAME, MIN(sizeof(shmcon->name), strlen(SHMCON_NAME)));
 469 #pragma clang diagnostic pop
 470                 for (i = 0; i < NUM_CHILDREN; i++) {
 471                         shmcon->child[0] = 0;
 472                 }
 473                 shmcon_barrier();
 474                 shmcon->magic = SHMCON_MAGIC;
 475         }
 476         end =  (volatile struct shm_buffer_info *)va_buffer_end;
 477         end->base = pa_panic_base;
 478         end->unused = 0;
 479         shmcon_barrier();
 480         end->magic = SHMCON_MAGIC;
 481 #ifdef SHMCON_THROTTLED
 482         grace = gPEClockFrequencyInfo.timebase_frequency_hz;
 483 #endif
 484
 485         PE_consistent_debug_register(kDbgIdConsoleHeaderAP, pa_panic_base, panic_size);
 486 }
 487
 488 SECURITY_READ_ONLY_LATE(static struct pe_serial_functions) shmcon_serial_functions =
 489 {
 490         .uart_init = shmcon_init,
 491         .uart_set_baud_rate = shmcon_set_baud_rate,
 492         .tr0 = shmcon_tr0,
 493         .td0 = shmcon_td0,
 494         .rr0 = shmcon_rr0,
 495         .rd0 = shmcon_rd0
 496 };
 497
 498 int
 499 pe_shmcon_set_child(uint64_t paddr, uint32_t entry)
 500 {
 501         if (shmcon == NULL) {
 502                 return -1;
 503         }
 504
 505         if (shmcon->children >= entry) {
 506                 return -1;
 507         }
 508
 509         shmcon->child[entry] = paddr;
 510         return 0;
 511 }
 512
 513 #endif /* SHMCON */
 514
 515 /*****************************************************************************/
 516
 517 #ifdef DOCKCHANNEL_UART
 518 #define DOCKCHANNEL_WR_MAX_STALL_US     (30*1000)
 519
 520 static vm_offset_t      dock_agent_base;
 521 static uint32_t         max_dockchannel_drain_period;
 522 static bool             use_sw_drain;
 523 static uint32_t         dock_wstat_mask;
 524 static uint64_t         prev_dockchannel_drained_time;  // Last time we've seen the DockChannel drained by an external agent
 525 static uint64_t         prev_dockchannel_spaces;        // Previous w_stat level of the DockChannel.
 526 static uint64_t         dockchannel_stall_grace;
 527 static vm_offset_t      dockchannel_uart_base = 0;
 528
 529 //=======================
 530 // Local funtions
 531 //=======================
 532
 533 static int
 534 dockchannel_drain_on_stall()
 535 {
 536         // Called when DockChannel runs out of spaces.
 537         // Check if the DockChannel reader has stalled. If so, empty the DockChannel ourselves.
 538         // Return number of bytes drained.
 539
 540         if ((mach_absolute_time() - prev_dockchannel_drained_time) >= dockchannel_stall_grace) {
 541                 // It's been more than DOCKCHANEL_WR_MAX_STALL_US and nobody read from the FIFO
 542                 // Drop a character.
 543                 (void)rDOCKCHANNELS_DOCK_RDATA1(DOCKCHANNEL_UART_CHANNEL);
 544                 os_atomic_inc(&prev_dockchannel_spaces, relaxed);
 545                 return 1;
 546         }
 547         return 0;
 548 }
 549
 550 static int
 551 dockchannel_uart_tr0(void)
 552 {
 553         if (use_sw_drain) {
 554                 uint32_t spaces = rDOCKCHANNELS_DEV_WSTAT(DOCKCHANNEL_UART_CHANNEL) & dock_wstat_mask;
 555                 if (spaces > prev_dockchannel_spaces) {
 556                         // More spaces showed up. That can only mean someone read the FIFO.
 557                         // Note that if the DockFIFO is empty we cannot tell if someone is listening,
 558                         // we can only give them the benefit of the doubt.
 559                         prev_dockchannel_drained_time = mach_absolute_time();
 560                 }
 561                 prev_dockchannel_spaces = spaces;
 562
 563                 return spaces || dockchannel_drain_on_stall();
 564         } else {
 565                 // Returns spaces in dockchannel fifo
 566                 return rDOCKCHANNELS_DEV_WSTAT(DOCKCHANNEL_UART_CHANNEL) & dock_wstat_mask;
 567         }
 568 }
 569
 570 static void
 571 dockchannel_uart_td0(int c)
 572 {
 573         rDOCKCHANNELS_DEV_WDATA1(DOCKCHANNEL_UART_CHANNEL) = (unsigned)(c & 0xff);
 574         if (use_sw_drain) {
 575                 os_atomic_dec(&prev_dockchannel_spaces, relaxed); // After writing a byte we have one fewer space than previously expected.
 576         }
 577 }
 578
 579 static int
 580 dockchannel_uart_rr0(void)
 581 {
 582         return rDOCKCHANNELS_DEV_RDATA0(DOCKCHANNEL_UART_CHANNEL) & 0x7f;
 583 }
 584
 585 static int
 586 dockchannel_uart_rd0(void)
 587 {
 588         return (int)((rDOCKCHANNELS_DEV_RDATA1(DOCKCHANNEL_UART_CHANNEL) >> 8) & 0xff);
 589 }
 590
 591 static void
 592 dockchannel_uart_clear_intr(void)
 593 {
 594         rDOCKCHANNELS_AGENT_AP_INTR_CTRL &= ~(0x3);
 595         rDOCKCHANNELS_AGENT_AP_INTR_STATUS |= 0x3;
 596         rDOCKCHANNELS_AGENT_AP_ERR_INTR_CTRL &= ~(0x3);
 597         rDOCKCHANNELS_AGENT_AP_ERR_INTR_STATUS |= 0x3;
 598 }
 599
 600 static void
 601 dockchannel_uart_init(void)
 602 {
 603         if (use_sw_drain) {
 604                 nanoseconds_to_absolutetime(DOCKCHANNEL_WR_MAX_STALL_US * NSEC_PER_USEC, &dockchannel_stall_grace);
 605         }
 606
 607         // Clear all interrupt enable and status bits
 608         dockchannel_uart_clear_intr();
 609
 610         // Setup DRAIN timer
 611         rDOCKCHANNELS_DEV_DRAIN_CFG(DOCKCHANNEL_UART_CHANNEL) = max_dockchannel_drain_period;
 612
 613         // Drain timer doesn't get loaded with value from drain period register if fifo
 614         // is already full. Drop a character from the fifo.
 615         rDOCKCHANNELS_DOCK_RDATA1(DOCKCHANNEL_UART_CHANNEL);
 616 }
 617
 618 SECURITY_READ_ONLY_LATE(static struct pe_serial_functions) dockchannel_uart_serial_functions =
 619 {
 620         .uart_init = dockchannel_uart_init,
 621         .uart_set_baud_rate = NULL,
 622         .tr0 = dockchannel_uart_tr0,
 623         .td0 = dockchannel_uart_td0,
 624         .rr0 = dockchannel_uart_rr0,
 625         .rd0 = dockchannel_uart_rd0
 626 };
 627
 628 #endif /* DOCKCHANNEL_UART */
 629
 630 /****************************************************************************/
 631 #ifdef  PI3_UART
 632 static vm_offset_t pi3_gpio_base_vaddr = 0;
 633 static vm_offset_t pi3_aux_base_vaddr = 0;
 634 static int
 635 pi3_uart_tr0(void)
 636 {
 637         return (int) BCM2837_GET32(BCM2837_AUX_MU_LSR_REG_V) & 0x20;
 638 }
 639
 640 static void
 641 pi3_uart_td0(int c)
 642 {
 643         BCM2837_PUT32(BCM2837_AUX_MU_IO_REG_V, (uint32_t) c);
 644 }
 645
 646 static int
 647 pi3_uart_rr0(void)
 648 {
 649         return (int) BCM2837_GET32(BCM2837_AUX_MU_LSR_REG_V) & 0x01;
 650 }
 651
 652 static int
 653 pi3_uart_rd0(void)
 654 {
 655         return (int) BCM2837_GET32(BCM2837_AUX_MU_IO_REG_V) & 0xff;
 656 }
 657
 658 static void
 659 pi3_uart_init(void)
 660 {
 661         // Scratch variable
 662         uint32_t i;
 663
 664         // Reset mini uart registers
 665         BCM2837_PUT32(BCM2837_AUX_ENABLES_V, 1);
 666         BCM2837_PUT32(BCM2837_AUX_MU_CNTL_REG_V, 0);
 667         BCM2837_PUT32(BCM2837_AUX_MU_LCR_REG_V, 3);
 668         BCM2837_PUT32(BCM2837_AUX_MU_MCR_REG_V, 0);
 669         BCM2837_PUT32(BCM2837_AUX_MU_IER_REG_V, 0);
 670         BCM2837_PUT32(BCM2837_AUX_MU_IIR_REG_V, 0xC6);
 671         BCM2837_PUT32(BCM2837_AUX_MU_BAUD_REG_V, 270);
 672
 673         i = (uint32_t)BCM2837_FSEL_REG(14);
 674         // Configure GPIOs 14 & 15 for alternate function 5
 675         i &= ~(BCM2837_FSEL_MASK(14));
 676         i |= (BCM2837_FSEL_ALT5 << BCM2837_FSEL_OFFS(14));
 677         i &= ~(BCM2837_FSEL_MASK(15));
 678         i |= (BCM2837_FSEL_ALT5 << BCM2837_FSEL_OFFS(15));
 679
 680         BCM2837_PUT32(BCM2837_FSEL_REG(14), i);
 681
 682         BCM2837_PUT32(BCM2837_GPPUD_V, 0);
 683
 684         // Barrier before AP spinning for 150 cycles
 685         __builtin_arm_isb(ISB_SY);
 686
 687         for (i = 0; i < 150; i++) {
 688                 asm volatile ("add x0, x0, xzr");
 689         }
 690
 691         __builtin_arm_isb(ISB_SY);
 692
 693         BCM2837_PUT32(BCM2837_GPPUDCLK0_V, (1 << 14) | (1 << 15));
 694
 695         __builtin_arm_isb(ISB_SY);
 696
 697         for (i = 0; i < 150; i++) {
 698                 asm volatile ("add x0, x0, xzr");
 699         }
 700
 701         __builtin_arm_isb(ISB_SY);
 702
 703         BCM2837_PUT32(BCM2837_GPPUDCLK0_V, 0);
 704
 705         BCM2837_PUT32(BCM2837_AUX_MU_CNTL_REG_V, 3);
 706 }
 707
 708 SECURITY_READ_ONLY_LATE(static struct pe_serial_functions) pi3_uart_serial_functions =
 709 {
 710         .uart_init = pi3_uart_init,
 711         .uart_set_baud_rate = NULL,
 712         .tr0 = pi3_uart_tr0,
 713         .td0 = pi3_uart_td0,
 714         .rr0 = pi3_uart_rr0,
 715         .rd0 = pi3_uart_rd0
 716 };
 717
 718 #endif /* PI3_UART */
 719
 720 /*****************************************************************************/
 721
 722
 723 /*****************************************************************************/
 724
 725 static void
 726 register_serial_functions(struct pe_serial_functions *fns)
 727 {
 728         fns->next = gPESF;
 729         gPESF = fns;
 730 }
 731
 732 int
 733 serial_init(void)
 734 {
 735         DTEntry         entryP = NULL;
 736         uint32_t        prop_size;
 737         vm_offset_t     soc_base;
 738         uintptr_t const *reg_prop;
 739         uint32_t const  *prop_value __unused = NULL;
 740         char const      *serial_compat __unused = 0;
 741         uint32_t        dccmode;
 742
 743         struct pe_serial_functions *fns = gPESF;
 744
 745         if (uart_initted) {
 746                 while (fns != NULL) {
 747                         fns->uart_init();
 748                         fns = fns->next;
 749                 }
 750                 kprintf("reinit serial\n");
 751                 return 1;
 752         }
 753
 754         dccmode = 0;
 755         if (PE_parse_boot_argn("dcc", &dccmode, sizeof(dccmode))) {
 756                 register_serial_functions(&dcc_serial_functions);
 757         }
 758 #ifdef SHMCON
 759         uint32_t jconmode = 0;
 760         if (PE_parse_boot_argn("jcon", &jconmode, sizeof jconmode)) {
 761                 register_serial_functions(&shmcon_serial_functions);
 762         }
 763 #endif /* SHMCON */
 764
 765         soc_base = pe_arm_get_soc_base_phys();
 766
 767         if (soc_base == 0) {
 768                 return 0;
 769         }
 770
 771 #ifdef PI3_UART
 772         if (SecureDTFindEntry("name", "gpio", &entryP) == kSuccess) {
 773                 SecureDTGetProperty(entryP, "reg", (void const **)&reg_prop, &prop_size);
 774                 pi3_gpio_base_vaddr = ml_io_map(soc_base + *reg_prop, *(reg_prop + 1));
 775         }
 776         if (SecureDTFindEntry("name", "aux", &entryP) == kSuccess) {
 777                 SecureDTGetProperty(entryP, "reg", (void const **)&reg_prop, &prop_size);
 778                 pi3_aux_base_vaddr = ml_io_map(soc_base + *reg_prop, *(reg_prop + 1));
 779         }
 780         if ((pi3_gpio_base_vaddr != 0) && (pi3_aux_base_vaddr != 0)) {
 781                 register_serial_functions(&pi3_uart_serial_functions);
 782         }
 783 #endif /* PI3_UART */
 784
 785
 786 #ifdef DOCKCHANNEL_UART
 787         uint32_t no_dockchannel_uart = 0;
 788         if (SecureDTFindEntry("name", "dockchannel-uart", &entryP) == kSuccess) {
 789                 SecureDTGetProperty(entryP, "reg", (void const **)&reg_prop, &prop_size);
 790                 // Should be two reg entries
 791                 if (prop_size / sizeof(uintptr_t) != 4) {
 792                         panic("Malformed dockchannel-uart property");
 793                 }
 794                 dockchannel_uart_base = ml_io_map(soc_base + *reg_prop, *(reg_prop + 1));
 795                 dock_agent_base = ml_io_map(soc_base + *(reg_prop + 2), *(reg_prop + 3));
 796                 PE_parse_boot_argn("no-dockfifo-uart", &no_dockchannel_uart, sizeof(no_dockchannel_uart));
 797                 // Keep the old name for boot-arg
 798                 if (no_dockchannel_uart == 0) {
 799                         register_serial_functions(&dockchannel_uart_serial_functions);
 800                         SecureDTGetProperty(entryP, "max-aop-clk", (void const **)&prop_value, &prop_size);
 801                         max_dockchannel_drain_period = (uint32_t)((prop_value)?  (*prop_value * 0.03) : DOCKCHANNEL_DRAIN_PERIOD);
 802                         prop_value = NULL;
 803                         SecureDTGetProperty(entryP, "enable-sw-drain", (void const **)&prop_value, &prop_size);
 804                         use_sw_drain = (prop_value)?  *prop_value : 0;
 805                         prop_value = NULL;
 806                         SecureDTGetProperty(entryP, "dock-wstat-mask", (void const **)&prop_value, &prop_size);
 807                         dock_wstat_mask = (prop_value)?  *prop_value : 0x1ff;
 808                 } else {
 809                         dockchannel_uart_clear_intr();
 810                 }
 811                 // If no dockchannel-uart is found in the device tree, fall back
 812                 // to looking for the traditional UART serial console.
 813         }
 814
 815 #endif /* DOCKCHANNEL_UART */
 816
 817         /*
 818          * The boot serial port should have a property named "boot-console".
 819          * If we don't find it there, look for "uart0" and "uart1".
 820          */
 821
 822         if (SecureDTFindEntry("boot-console", NULL, &entryP) == kSuccess) {
 823                 SecureDTGetProperty(entryP, "reg", (void const **)&reg_prop, &prop_size);
 824                 uart_base = ml_io_map(soc_base + *reg_prop, *(reg_prop + 1));
 825                 if (serial_compat == 0) {
 826                         SecureDTGetProperty(entryP, "compatible", (void const **)&serial_compat, &prop_size);
 827                 }
 828         } else if (SecureDTFindEntry("name", "uart0", &entryP) == kSuccess) {
 829                 SecureDTGetProperty(entryP, "reg", (void const **)&reg_prop, &prop_size);
 830                 uart_base = ml_io_map(soc_base + *reg_prop, *(reg_prop + 1));
 831                 if (serial_compat == 0) {
 832                         SecureDTGetProperty(entryP, "compatible", (void const **)&serial_compat, &prop_size);
 833                 }
 834         } else if (SecureDTFindEntry("name", "uart1", &entryP) == kSuccess) {
 835                 SecureDTGetProperty(entryP, "reg", (void const **)&reg_prop, &prop_size);
 836                 uart_base = ml_io_map(soc_base + *reg_prop, *(reg_prop + 1));
 837                 if (serial_compat == 0) {
 838                         SecureDTGetProperty(entryP, "compatible", (void const **)&serial_compat, &prop_size);
 839                 }
 840         }
 841 #ifdef  S3CUART
 842         if (NULL != entryP) {
 843                 SecureDTGetProperty(entryP, "pclk", (void const **)&prop_value, &prop_size);
 844                 if (prop_value) {
 845                         dt_pclk = *prop_value;
 846                 }
 847
 848                 prop_value = NULL;
 849                 SecureDTGetProperty(entryP, "sampling", (void const **)&prop_value, &prop_size);
 850                 if (prop_value) {
 851                         dt_sampling = *prop_value;
 852                 }
 853
 854                 prop_value = NULL;
 855                 SecureDTGetProperty(entryP, "ubrdiv", (void const **)&prop_value, &prop_size);
 856                 if (prop_value) {
 857                         dt_ubrdiv = *prop_value;
 858                 }
 859         }
 860
 861         if (serial_compat) {
 862                 if (!strcmp(serial_compat, "uart,16550")) {
 863                         register_serial_functions(&ln2410_serial_functions);
 864                 } else if (!strcmp(serial_compat, "uart-16550")) {
 865                         register_serial_functions(&ln2410_serial_functions);
 866                 } else if (!strcmp(serial_compat, "uart,s5i3000")) {
 867                         register_serial_functions(&ln2410_serial_functions);
 868                 } else if (!strcmp(serial_compat, "uart-1,samsung")) {
 869                         register_serial_functions(&ln2410_serial_functions);
 870                 }
 871         }
 872 #endif /* S3CUART */
 873
 874         if (gPESF == NULL) {
 875                 return 0;
 876         }
 877
 878         fns = gPESF;
 879         while (fns != NULL) {
 880                 fns->uart_init();
 881                 fns = fns->next;
 882         }
 883
 884 #if HIBERNATION
 885         /* hibernation needs to know the UART register addresses since it can't directly use this serial driver */
 886         if (dockchannel_uart_base) {
 887                 gHibernateGlobals.dockChannelRegBase = ml_vtophys(dockchannel_uart_base);
 888                 gHibernateGlobals.dockChannelWstatMask = dock_wstat_mask;
 889         }
 890         if (uart_base) {
 891                 gHibernateGlobals.hibUartRegBase = ml_vtophys(uart_base);
 892         }
 893 #endif /* HIBERNATION */
 894
 895         uart_initted = 1;
 896
 897         return 1;
 898 }
 899
 900 void
 901 uart_putc(char c)
 902 {
 903         struct pe_serial_functions *fns = gPESF;
 904         while (fns != NULL) {
 905                 while (!fns->tr0()) {
 906 #if __arm64__                           /* on arm64, we have a WFE timeout, so no need to hot-poll here */
 907                         __builtin_arm_wfe()
 908 #endif
 909                         ;               /* Wait until THR is empty. */
 910                 }
 911                 fns->td0(c);
 912                 fns = fns->next;
 913         }
 914 }
 915
 916 int
 917 uart_getc(void)
 918 {                               /* returns -1 if no data available */
 919         struct pe_serial_functions *fns = gPESF;
 920         while (fns != NULL) {
 921                 if (fns->rr0()) {
 922                         return fns->rd0();
 923                 }
 924                 fns = fns->next;
 925         }
 926         return -1;
 927 }