Index: apps/debug_menu.c =================================================================== --- apps/debug_menu.c (revision 20280) +++ apps/debug_menu.c (working copy) @@ -2029,7 +2029,7 @@ simplelist_addline(SIMPLELIST_ADD_LINE, "No timing info"); } -#if defined (TOSHIBA_GIGABEAT_F) || defined (TOSHIBA_GIGABEAT_S) +#if defined (TOSHIBA_GIGABEAT_F) || defined (TOSHIBA_GIGABEAT_S) || defined (HAVE_ATA_DMA) if (identify_info[63] & (1<<0)) { char mdma0[2], mdma1[2], mdma2[2]; mdma0[1] = mdma1[1] = mdma2[1] = 0; @@ -2047,18 +2047,19 @@ simplelist_addline(SIMPLELIST_ADD_LINE, "No MDMA mode info"); } - if (identify_info[88] & (1<<0)) { - char udma0[2], udma1[2], udma2[2], udma3[2], udma4[2], udma5[2]; - udma0[1] = udma1[1] = udma2[1] = udma3[1] = udma4[1] = udma5[1] = 0; + if (identify_info[53] & (1<<2)) { + char udma0[2], udma1[2], udma2[2], udma3[2], udma4[2], udma5[2], udma6[2]; + udma0[1] = udma1[1] = udma2[1] = udma3[1] = udma4[1] = udma5[1] = udma6[1] = 0; udma0[0] = (identify_info[88] & (1<<0)) ? '0' : 0; udma1[0] = (identify_info[88] & (1<<1)) ? '1' : 0; udma2[0] = (identify_info[88] & (1<<2)) ? '2' : 0; udma3[0] = (identify_info[88] & (1<<3)) ? '3' : 0; udma4[0] = (identify_info[88] & (1<<4)) ? '4' : 0; udma5[0] = (identify_info[88] & (1<<5)) ? '5' : 0; + udma6[0] = (identify_info[88] & (1<<6)) ? '6' : 0; simplelist_addline(SIMPLELIST_ADD_LINE, - "UDMA modes: %s %s %s %s %s %s", udma0, udma1, udma2, - udma3, udma4, udma5); + "UDMA modes: %s %s %s %s %s %s %s", udma0, udma1, udma2, + udma3, udma4, udma5, udma6); } else { simplelist_addline(SIMPLELIST_ADD_LINE, @@ -2079,6 +2080,18 @@ } simplelist_addline(SIMPLELIST_ADD_LINE, "Cluster size: %d bytes", fat_get_cluster_size(IF_MV(0))); +#ifdef HAVE_ATA_DMA + i = ata_get_dma_mode(); + if (i == 0) { + simplelist_addline(SIMPLELIST_ADD_LINE, + "DMA not enabled"); + } else { + simplelist_addline(SIMPLELIST_ADD_LINE, + "DMA mode: %s %c", + (i & 0x40) ? "UDMA" : "MDMA", + '0' + (i & 7)); + } +#endif return btn; } #else /* No SD, MMC or ATA */ Index: firmware/export/config-ipodvideo.h =================================================================== --- firmware/export/config-ipodvideo.h (revision 20280) +++ firmware/export/config-ipodvideo.h (working copy) @@ -213,4 +213,6 @@ #define HAVE_LCD_SHUTDOWN #endif +/* Define this to add support for ATA DMA */ +#define HAVE_ATA_DMA #endif Index: firmware/export/ata.h =================================================================== --- firmware/export/ata.h (revision 20280) +++ firmware/export/ata.h (working copy) @@ -60,5 +60,10 @@ long ata_last_disk_activity(void); int ata_spinup_time(void); /* ticks */ +#ifdef HAVE_ATA_DMA +/* Needed to allow updating while waiting for DMA to complete */ +extern long last_disk_activity; +int ata_get_dma_mode(void); +#endif #endif Index: firmware/export/config-gigabeat-s.h =================================================================== --- firmware/export/config-gigabeat-s.h (revision 20280) +++ firmware/export/config-gigabeat-s.h (working copy) @@ -186,6 +186,11 @@ /* Define this if you have ATA power-off control */ #define HAVE_ATA_POWER_OFF +#ifndef BOOTLOADER +/* Define this to add support for ATA DMA */ +#define HAVE_ATA_DMA +#endif + /* Virtual LED (icon) */ #define CONFIG_LED LED_VIRTUAL Index: firmware/export/imx31l.h =================================================================== --- firmware/export/imx31l.h (revision 20280) +++ firmware/export/imx31l.h (working copy) @@ -497,11 +497,11 @@ #define ATA_IORDY_EN (1 << 0) /* ATA_INTERRUPT_PENDING, ATA_INTERRUPT_ENABLE, ATA_INTERRUPT_CLEAR flags */ -#define ATA_INTRQ1 (1 << 7) +#define ATA_INTRQ1 (1 << 7) /* INTRQ to the DMA */ #define ATA_FIFO_UNDERFLOW (1 << 6) #define ATA_FIFO_OVERFLOW (1 << 5) #define ATA_CONTROLLER_IDLE (1 << 4) -#define ATA_INTRQ2 (1 << 3) +#define ATA_INTRQ2 (1 << 3) /* INTRQ to the MCU */ /* EPIT */ #define EPITCR1 (*(REG32_PTR_T)(EPIT1_BASE_ADDR+0x00)) Index: firmware/target/arm/ata-target.h =================================================================== --- firmware/target/arm/ata-target.h (revision 20280) +++ firmware/target/arm/ata-target.h (working copy) @@ -82,3 +82,29 @@ void ata_enable(bool on); bool ata_is_coldstart(void); void ata_device_init(void); + +#ifdef HAVE_ATA_DMA + +/* IDE DMA controller registers */ +#define IDE_DMA_CONTROL (*(volatile unsigned long *)(0xc3000400)) +#define IDE_DMA_LENGTH (*(volatile unsigned long *)(0xc3000408)) +#define IDE_DMA_ADDR (*(volatile unsigned long *)(0xc300040C)) + +/* Maximum multi-word DMA mode supported by the controller */ +#define ATA_MAX_MWDMA 2 + +#ifndef BOOTLOADER +/* The PP5020 supports UDMA 4, but it needs cpu boosting and doesn't + improve test_disk results. UDMA 2 is stable at 30 Mhz. + */ +#define ATA_MAX_UDMA 1 /* AB: UDMA 1 for 24MHz normal clock */ +#else +/* The bootloader runs at 24 Mhz and needs a slower mode */ +#define ATA_MAX_UDMA 1 +#endif + +void ata_dma_set_mode(unsigned char mode); +bool ata_dma_setup(void *addr, unsigned long bytes, bool write); +bool ata_dma_finish(void); + +#endif /* HAVE_ATA_DMA */ Index: firmware/target/arm/imx31/gigabeat-s/ata-imx31.c =================================================================== --- firmware/target/arm/imx31/gigabeat-s/ata-imx31.c (revision 20280) +++ firmware/target/arm/imx31/gigabeat-s/ata-imx31.c (working copy) @@ -28,7 +28,13 @@ #include "ata.h" #include "ata-target.h" #include "clkctl-imx31.h" +#ifdef HAVE_ATA_DMA +#include +#include "sdma-imx31.h" +#include "mmu-imx31.h" +#endif +/* PIO modes timing info */ static const struct ata_pio_timings { uint16_t time_2w; /* t2 during write */ @@ -83,27 +89,207 @@ .time_ax = 35, .time_4 = 10, .time_9 = 10 + } +}; + +#ifdef HAVE_ATA_DMA +/* One DMA channel for reads, the other for writes */ +#define ATA_DMA_CH_NUM_RD 3 +#define ATA_DMA_CH_NUM_WR 4 +/* Use default priority for these channels (1) */ +/* Maximum DMA size per buffer descriptor (32-byte aligned) */ +#define ATA_MAX_BD_SIZE (65534 & ~31) /* 65504 */ + +/* Number of buffer descriptors required for a maximum sector count trasfer. + * NOTE: Assumes LBA28 and 512-byte sectors! */ +#define ATA_BASE_BD_COUNT ((256*512 + (ATA_MAX_BD_SIZE-1)) / ATA_MAX_BD_SIZE) +#define ATA_BD_COUNT (ATA_BASE_BD_COUNT + 2) + +static const struct ata_mdma_timings +{ + uint8_t time_m; /* tM */ + uint8_t time_jn; /* tH */ + uint8_t time_d; /* tD */ + uint8_t time_k; /* tKW */ +} mdma_timings[] = +{ + [0] = /* MDMA mode 0 */ + { + .time_m = 50, + .time_jn = 20, + .time_d = 215, + .time_k = 215 }, + [1] = /* MDMA mode 1 */ + { + .time_m = 30, + .time_jn = 15, + .time_d = 80, + .time_k = 50 + }, + [2] = /* MDMA mode 2 */ + { + .time_m = 25, + .time_jn = 10, + .time_d = 70, + .time_k = 25 + } }; -static int pio_mode = 0; /* Setup mode 0 by default */ +static const struct ata_udma_timings +{ + uint8_t time_ack; /* tACK */ + uint8_t time_env; /* tENV */ + uint8_t time_rpx; /* tRP */ + uint8_t time_zah; /* tZAH */ + uint8_t time_mlix; /* tMLI */ + uint8_t time_dvh; /* tDVH */ + uint8_t time_dzfs; /* tDVS+tDVH? */ + uint8_t time_dvs; /* tDVS */ + uint8_t time_cvh; /* ?? */ + uint8_t time_ss; /* tSS */ + uint8_t time_cyc; /* tCYC */ +} udma_timings[] = +{ + [0] = /* UDMA mode 0 */ + { + .time_ack = 20, + .time_env = 20, + .time_rpx = 160, + .time_zah = 20, + .time_mlix = 20, + .time_dvh = 6, + .time_dzfs = 80, + .time_dvs = 70, + .time_cvh = 6, + .time_ss = 50, + .time_cyc = 114 + }, + [1] = /* UDMA mode 1 */ + { + .time_ack = 20, + .time_env = 20, + .time_rpx = 125, + .time_zah = 20, + .time_mlix = 20, + .time_dvh = 6, + .time_dzfs = 63, + .time_dvs = 48, + .time_cvh = 6, + .time_ss = 50, + .time_cyc = 75 + }, + [2] = /* UDMA mode 2 */ + { + .time_ack = 20, + .time_env = 20, + .time_rpx = 100, + .time_zah = 20, + .time_mlix = 20, + .time_dvh = 6, + .time_dzfs = 47, + .time_dvs = 34, + .time_cvh = 6, + .time_ss = 50, + .time_cyc = 55 + }, + [3] = /* UDMA mode 3 */ + { + .time_ack = 20, + .time_env = 20, + .time_rpx = 100, + .time_zah = 20, + .time_mlix = 20, + .time_dvh = 6, + .time_dzfs = 35, + .time_dvs = 20, + .time_cvh = 6, + .time_ss = 50, + .time_cyc = 39 + }, + [4] = /* UDMA mode 4 */ + { + .time_ack = 20, + .time_env = 20, + .time_rpx = 100, + .time_zah = 20, + .time_mlix = 20, + .time_dvh = 6, + .time_dzfs = 25, + .time_dvs = 7, + .time_cvh = 6, + .time_ss = 50, + .time_cyc = 25 + }, +#if 0 + [5] = /* UDMA mode 5 (bus clock 80MHz or higher only) */ + { + .time_ack = 20, + .time_env = 20, + .time_rpx = 85, + .time_zah = 20, + .time_mlix = 20, + .time_dvh = 6, + .time_dzfs = 40, + .time_dvs = 5, + .time_cvh = 10, + .time_ss = 50, + .time_cyc = 17 + } +#endif +}; +/* Track first init */ +static bool initialized = false; +/* Signal to tell thread when DMA is done */ +static struct wakeup ata_dma_wakeup; +/* Array of buffer descriptors for large transfers and alignnment */ +static struct buffer_descriptor ata_bda[ATA_BD_COUNT] DEVBSS_ATTR; +/* ATA channel descriptors */ +static struct channel_descriptor ata_cd_rd DEVBSS_ATTR; /* read channel */ +static struct channel_descriptor ata_cd_wr DEVBSS_ATTR; /* write channel */ +/* Scatter buffer for first and last lines of a non cache-aligned transfer + * to/from cached RAM. */ +static unsigned char scatter_buffer[2][32] + __attribute__((aligned(4))) DEVBSS_ATTR; +/* Address of ends in destination buffer for unaligned reads - copied after + * DMA completes. Not used for transfers to uncached buffers regardless of + * alignment. */ +static unsigned char *sb_dst[2] = { NULL, NULL }; +/* Current DMA channel */ +static unsigned int current_channel = 0; +/* ATA_DMA_ULTRA_SELECTED bit, ATA_DMA_OFF if DMA was never set up or + * setup wasn't understood. */ +#define ATA_DMA_OFF (~0ul) +static unsigned long ata_dma_ultra_selected = ATA_DMA_OFF; +#endif /* HAVE_ATA_DMA */ + +static unsigned int get_T(void) +{ + /* T = ATA clock period in nanoseconds */ + return 1000 * 1000 * 1000 / imx31_clkctl_get_ata_clk(); +} + static void ata_wait_for_idle(void) { while (!(ATA_INTERRUPT_PENDING & ATA_CONTROLLER_IDLE)); } +static inline void ata_set_intrq(bool to_dma) +{ + ATA_INTERRUPT_ENABLE = + (ATA_INTERRUPT_ENABLE & ~(ATA_INTRQ1 | ATA_INTRQ2)) | + (to_dma ? ATA_INTRQ1 : ATA_INTRQ2); +} + /* Setup the timing for PIO mode */ void ata_set_pio_timings(int mode) { ata_wait_for_idle(); const struct ata_pio_timings * const timings = &pio_timings[mode]; - /* T = period in nanoseconds */ - int T = 1000 * 1000 * 1000 / imx31_clkctl_get_ata_clk(); + unsigned int T = get_T(); - pio_mode = mode; - ATA_TIME_1 = (timings->time_1 + T) / T; ATA_TIME_2W = (timings->time_2w + T) / T; ATA_TIME_2R = (timings->time_2r + T) / T; @@ -118,10 +304,10 @@ /* Be sure we're not busy */ ata_wait_for_idle(); - ATA_INTF_CONTROL &= ~ATA_ATA_RST; - sleep(1); - ATA_INTF_CONTROL |= ATA_ATA_RST; - sleep(1); + ATA_INTF_CONTROL &= ~(ATA_ATA_RST | ATA_FIFO_RST); + sleep(HZ/100); + ATA_INTF_CONTROL = ATA_ATA_RST | ATA_FIFO_RST; + sleep(HZ/100); ata_wait_for_idle(); } @@ -130,17 +316,18 @@ { /* Unconditionally clock module before writing regs */ imx31_clkctl_module_clock_gating(CG_ATA, CGM_ON_ALL); + ata_wait_for_idle(); if (on) { - ATA_INTF_CONTROL |= ATA_ATA_RST; + ATA_INTF_CONTROL = ATA_ATA_RST | ATA_FIFO_RST; + sleep(HZ/100); } else { - ata_wait_for_idle(); + ATA_INTF_CONTROL &= ~(ATA_ATA_RST | ATA_FIFO_RST); + sleep(HZ/100); - ATA_INTF_CONTROL &= ~ATA_ATA_RST; - /* Disable off - unclock ATA module */ imx31_clkctl_module_clock_gating(CG_ATA, CGM_OFF); } @@ -151,16 +338,331 @@ return true; } +#ifdef HAVE_ATA_DMA +static void ata_set_mdma_timings(unsigned int mode) +{ + ata_wait_for_idle(); + + const struct ata_mdma_timings * const timings = &mdma_timings[mode]; + unsigned int T = get_T(); + + ATA_TIME_M = (timings->time_m + T) / T; + ATA_TIME_JN = (timings->time_jn + T) / T; + ATA_TIME_D = (timings->time_d + T) / T; + ATA_TIME_K = (timings->time_k + T) / T; +} + +static void ata_set_udma_timings(unsigned int mode) +{ + const struct ata_udma_timings * const timings = &udma_timings[mode]; + unsigned int T = get_T(); + + ATA_TIME_ACK = (timings->time_ack + T) / T; + ATA_TIME_ENV = (timings->time_env + T) / T; + ATA_TIME_RPX = (timings->time_rpx + T) / T; + ATA_TIME_ZAH = (timings->time_zah + T) / T; + ATA_TIME_MLIX = (timings->time_mlix + T) / T; + ATA_TIME_DVH = (timings->time_dvh + T) / T + 1; + ATA_TIME_DZFS = (timings->time_dzfs + T) / T; + ATA_TIME_DVS = (timings->time_dvs + T) / T; + ATA_TIME_CVH = (timings->time_cvh + T) / T; + ATA_TIME_SS = (timings->time_ss + T) / T; + ATA_TIME_CYC = (timings->time_cyc + T) / T; +} + +void ata_dma_set_mode(unsigned char mode) +{ + unsigned int modeidx = mode & 0x07; + unsigned int dmamode = mode & 0xf8; + + ata_wait_for_idle(); + + if (dmamode == 0x40 && modeidx <= ATA_MAX_UDMA) + { + /* Using Ultra DMA */ + ata_set_udma_timings(dmamode); + ata_dma_ultra_selected = ATA_DMA_ULTRA_SELECTED; + } + else if (dmamode == 0x20 && modeidx <= ATA_MAX_MWDMA) + { + /* Using Multiword DMA */ + ata_set_mdma_timings(dmamode); + ata_dma_ultra_selected = 0; + } + else + { + /* Don't understand this - force PIO. */ + ata_dma_ultra_selected = ATA_DMA_OFF; + } +} + +/* Called by SDMA when transfer is complete */ +static void ata_dma_callback(void) +{ + /* Clear FIFO if not empty - shouldn't happen */ + while (ATA_FIFO_FILL != 0) + ATA_FIFO_DATA_32; + + ATA_INTERRUPT_CLEAR = ATA_INTERRUPT_PENDING; + + ata_set_intrq(false); /* Return INTRQ to MCU */ + wakeup_signal(&ata_dma_wakeup); /* Signal waiting thread */ +} + +bool ata_dma_setup(void *addr, unsigned long bytes, bool write) +{ + struct buffer_descriptor *bd_p; + unsigned char *buf; + + if (bytes > ATA_BASE_BD_COUNT*ATA_MAX_BD_SIZE || + ata_dma_ultra_selected == ATA_DMA_OFF) + { + /* Too much? Implies BD count should be reevaluated since this + * shouldn't be reached based upon size. Otherwise we simply didn't + * understand the DMA mode setup. Force PIO in both cases. */ + ATA_INTF_CONTROL = ATA_FIFO_RST | ATA_ATA_RST; + return false; + } + + bd_p = &ata_bda[0]; + buf = (unsigned char *)addr_virt_to_phys((unsigned long)addr); + + if (write) + { + /* No cache alignment concerns */ + current_channel = ATA_DMA_CH_NUM_WR; + + if (buf != addr) + { + /* addr is virtual */ + clean_dcache_range(addr, bytes); + } + + /* Setup ATA controller for DMA transmit */ + ATA_INTF_CONTROL = ATA_FIFO_RST | ATA_ATA_RST | ATA_FIFO_TX_EN | + ATA_DMA_PENDING | ata_dma_ultra_selected | ATA_DMA_WRITE; + ATA_FIFO_ALARM = SDMA_ATA_WML / 2; + } + else + { + current_channel = ATA_DMA_CH_NUM_RD; + + /* Setup ATA controller for DMA receive */ + ATA_INTF_CONTROL = ATA_FIFO_RST | ATA_ATA_RST | ATA_FIFO_RCV_EN | + ATA_DMA_PENDING | ata_dma_ultra_selected; + ATA_FIFO_ALARM = SDMA_ATA_WML / 2; + + if (buf != addr) + { + /* addr is virtual */ + dump_dcache_range(addr, bytes); + + if ((unsigned long)addr & 31) + { + /* Not cache aligned, must use scatter buffers for first and + * last 32 bytes. */ + sb_dst[0] = addr; + bd_p->buf_addr = scatter_buffer[0]; + bd_p->mode.count = 32; + bd_p->mode.status = BD_DONE | BD_CONT; + + buf += 32; + addr += 32; + bytes -= 32; + bd_p++; + + while (bytes > ATA_MAX_BD_SIZE) + { + bd_p->buf_addr = buf; + bd_p->mode.count = ATA_MAX_BD_SIZE; + bd_p->mode.status = BD_DONE | BD_CONT; + buf += ATA_MAX_BD_SIZE; + addr += ATA_MAX_BD_SIZE; + bytes -= ATA_MAX_BD_SIZE; + bd_p++; + } + + if (bytes > 32) + { + bd_p->buf_addr = buf; + bd_p->mode.count = bytes - 32; + bd_p->mode.status = BD_DONE | BD_CONT; + addr += bytes - 32; + bd_p++; + } + + /* There will be exactly 32 bytes left */ + + /* Final buffer - wrap to base bd, interrupt */ + sb_dst[1] = addr; + bd_p->buf_addr = scatter_buffer[1]; + bd_p->mode.count = 32; + bd_p->mode.status = BD_DONE | BD_WRAP | BD_INTR; + + return true; + } + } + + sb_dst[0] = sb_dst[1] = NULL; /* No copies needed */ + } + + /* Setup buffer descriptors for both cache-aligned reads and all write + * operations. */ + while (bytes > ATA_MAX_BD_SIZE) + { + bd_p->buf_addr = buf; + bd_p->mode.count = ATA_MAX_BD_SIZE; + bd_p->mode.status = BD_DONE | BD_CONT; + buf += ATA_MAX_BD_SIZE; + bytes -= ATA_MAX_BD_SIZE; + bd_p++; + } + + /* Final buffer - wrap to base bd, interrupt */ + bd_p->buf_addr = buf; + bd_p->mode.count = bytes; + bd_p->mode.status = BD_DONE | BD_WRAP | BD_INTR; + + return true; +} + +bool ata_dma_finish(void) +{ + unsigned int channel = current_channel; + + ata_set_intrq(true); /* Give INTRQ to DMA */ + sdma_channel_run(channel); /* Kick the channel to wait for events */ + + current_channel = 0; + + if (UNLIKELY(wakeup_wait(&ata_dma_wakeup, HZ*10) != OBJ_WAIT_SUCCEEDED)) + { + /* Epic fail - timed out - maybe. */ + int oldirq = disable_irq_save(); + ata_set_intrq(false); /* Strip INTRQ from DMA */ + sdma_channel_stop(channel); /* Stop DMA */ + restore_irq(oldirq); + + if (wakeup_wait(&ata_dma_wakeup, TIMEOUT_NOBLOCK) != OBJ_WAIT_SUCCEEDED) + { + sdma_channel_reset(channel); /* Reset everything + clear error */ + return false; + } + + /* else DMA really did finish after timeout */ + } + + if (sdma_channel_is_error(channel)) + { + /* Channel error in one or more descriptors */ + sdma_channel_reset(channel); /* Reset everything + clear error */ + return false; + } + + if (channel == ATA_DMA_CH_NUM_RD) + { + /* NOTE: This requires that unaligned access support be enabled! */ + register void *sbs; + register void *sbd; + + if (sb_dst[0] != NULL) + { + sbs = scatter_buffer[0]; + sbd = sb_dst[0]; + asm volatile( + "add r1, %1, #32 \n" /* Prefetch at DMA-direct boundary */ + "mcrr p15, 2, r1, r1, c12 \n" + "ldmia %0!, { r0-r3 } \n" /* Copy the 32-bytes to destination */ + "str r0, [%1], #4 \n" /* stmia doesn't work unaligned */ + "str r1, [%1], #4 \n" + "str r2, [%1], #4 \n" + "str r3, [%1], #4 \n" + "ldmia %0!, { r0-r3 } \n" + "str r0, [%1], #4 \n" + "str r1, [%1], #4 \n" + "str r2, [%1], #4 \n" + "str r3, [%1] \n" + : "+r"(sbs), "+r"(sbd) + : + : "r0", "r1", "r2", "r3"); + } + + if (sb_dst[1] != NULL) + { + sbs = scatter_buffer[1]; + sbd = sb_dst[1]; + asm volatile( + "mcrr p15, 2, %1, %1, c12 \n" /* Prefetch at DMA-direct boundary */ + "ldmia %0!, { r0-r3 } \n" /* Copy the 32-bytes to destination */ + "str r0, [%1], #4 \n" /* stmia doesn't work unaligned */ + "str r1, [%1], #4 \n" + "str r2, [%1], #4 \n" + "str r3, [%1], #4 \n" + "ldmia %0!, { r0-r3 } \n" + "str r0, [%1], #4 \n" + "str r1, [%1], #4 \n" + "str r2, [%1], #4 \n" + "str r3, [%1] \n" + : "+r"(sbs), "+r"(sbd) + : + : "r0", "r1", "r2", "r3"); + } + } + + return true; +} +#endif /* HAVE_ATA_DMA */ + void ata_device_init(void) { +#ifdef HAVE_ATA_DMA + if (!initialized) + { + /* Called for first time at startup */ + wakeup_init(&ata_dma_wakeup); + + /* Read/write channels share buffer descriptors */ + ata_cd_rd.bd_count = ATA_BD_COUNT; + ata_cd_rd.callback = ata_dma_callback; + ata_cd_rd.shp_addr = SDMA_PER_ADDR_ATA_RX; + ata_cd_rd.wml = SDMA_ATA_WML; + ata_cd_rd.per_type = SDMA_PER_ATA; + ata_cd_rd.tran_type = SDMA_TRAN_PER_2_EMI; + ata_cd_rd.event_id1 = SDMA_REQ_ATA_TXFER_END; + ata_cd_rd.event_id2 = SDMA_REQ_ATA_RX; + + ata_cd_wr.bd_count = ATA_BD_COUNT; + ata_cd_wr.callback = ata_dma_callback; + ata_cd_wr.shp_addr = SDMA_PER_ADDR_ATA_TX; + ata_cd_wr.wml = SDMA_ATA_WML; + ata_cd_wr.per_type = SDMA_PER_ATA; + ata_cd_wr.tran_type = SDMA_TRAN_EMI_2_PER; + ata_cd_wr.event_id1 = SDMA_REQ_ATA_TXFER_END; + ata_cd_wr.event_id2 = SDMA_REQ_ATA_TX; + + sdma_channel_init(ATA_DMA_CH_NUM_RD, &ata_cd_rd, ata_bda); + sdma_channel_init(ATA_DMA_CH_NUM_WR, &ata_cd_wr, ata_bda); + initialized = true; + } +#endif /* HAVE_ATA_DMA */ + /* Make sure we're not in reset mode */ ata_enable(true); + ata_set_intrq(false); + /* All modes use same tOFF/tON */ ATA_TIME_OFF = 3; ATA_TIME_ON = 3; - /* mode may be switched later once identify info is ready in which + /* Mode may be switched later once identify info is ready in which * case the main driver calls back */ - ata_set_pio_timings(pio_mode); + + /* Setup mode 0 for all by default */ + ata_set_pio_timings(0); + +#ifdef HAVE_ATA_DMA + ata_set_mdma_timings(0); + ata_set_udma_timings(0); +#endif } Index: firmware/target/arm/imx31/gigabeat-s/ata-target.h =================================================================== --- firmware/target/arm/imx31/gigabeat-s/ata-target.h (revision 20280) +++ firmware/target/arm/imx31/gigabeat-s/ata-target.h (working copy) @@ -26,14 +26,19 @@ #endif /* Plain C read & write loops */ +/* They likely won't be used anyway since DMA potentially works for any + * sector number and alignment. */ #define PREFER_C_READING #define PREFER_C_WRITING -#if 0 -#if !defined(BOOTLOADER) -#define ATA_OPTIMIZED_WRITING -void copy_write_sectors(const unsigned char* buf, int wordcount); + +#ifdef HAVE_ATA_DMA +#define ATA_MAX_MWDMA 2 +#define ATA_MAX_UDMA 4 + +void ata_dma_set_mode(unsigned char mode); +bool ata_dma_setup(void *addr, unsigned long bytes, bool write); +bool ata_dma_finish(void); #endif -#endif #define ATA_DATA ATA_DRIVE_DATA #define ATA_ERROR ATA_DRIVE_FEATURES Index: firmware/target/arm/imx31/gigabeat-s/pcm-imx31.c =================================================================== --- firmware/target/arm/imx31/gigabeat-s/pcm-imx31.c (revision 20280) +++ firmware/target/arm/imx31/gigabeat-s/pcm-imx31.c (working copy) @@ -29,6 +29,8 @@ #define DMA_PLAY_CH_NUM 2 #define DMA_REC_CH_NUM 1 +#define DMA_PLAY_CH_PRIORITY 7 +#define DMA_REC_CH_PRIORITY 7 static struct buffer_descriptor dma_play_bd DEVBSS_ATTR; static struct channel_descriptor dma_play_cd DEVBSS_ATTR; @@ -126,6 +128,7 @@ dma_play_cd.event_id1 = SDMA_REQ_SSI2_TX1; sdma_channel_init(DMA_PLAY_CH_NUM, &dma_play_cd, &dma_play_bd); + sdma_channel_set_priority(DMA_PLAY_CH_NUM, DMA_PLAY_CH_PRIORITY); imx31_clkctl_module_clock_gating(CG_SSI1, CGM_ON_ALL); imx31_clkctl_module_clock_gating(CG_SSI2, CGM_ON_ALL); @@ -518,6 +521,7 @@ dma_rec_cd.event_id1 = SDMA_REQ_SSI1_RX1; sdma_channel_init(DMA_REC_CH_NUM, &dma_rec_cd, &dma_rec_bd); + sdma_channel_set_priority(DMA_REC_CH_NUM, DMA_REC_CH_PRIORITY); } const void * pcm_rec_dma_get_peak_buffer(int *count) Index: firmware/target/arm/imx31/sdma-imx31.c =================================================================== --- firmware/target/arm/imx31/sdma-imx31.c (revision 20280) +++ firmware/target/arm/imx31/sdma-imx31.c (working copy) @@ -468,6 +468,8 @@ CHANNEL_CONTEXT_ADDR(channel), sizeof (context_buffer)/4); + ccb_p->status.error = 0; /* Clear channel-wide error flag */ + if (cd_p->is_setup != 0) return true; /* No more to do */ @@ -759,6 +761,12 @@ memset(ccb_p, 0x00, sizeof (struct channel_control_block)); } +/* Check channel-wide error flag */ +bool sdma_channel_is_error(unsigned int channel) +{ + return channel < CH_NUM && ccb_array[channel].status.error; +} + /* Write 32-bit words to SDMA core memory. Host endian->SDMA endian. */ void sdma_write_words(const unsigned long *buf, unsigned long start, int count) { Index: firmware/target/arm/imx31/sdma-imx31.h =================================================================== --- firmware/target/arm/imx31/sdma-imx31.h (revision 20280) +++ firmware/target/arm/imx31/sdma-imx31.h (working copy) @@ -85,7 +85,7 @@ { SDMA_REQ_EXT0 = 0, /* Extern DMA request from MCU1_0 */ SDMA_REQ_CCM = 1, /* DVFS/DPTC event (ccm_dvfs_sdma_int) */ - SDMA_REQ_ATA_TX_END = 2, /* ata_txfer_end_alarm (event_id) */ + SDMA_REQ_ATA_TXFER_END = 2, /* ata_txfer_end_alarm (event_id) */ SDMA_REQ_ATA_TX = 3, /* ata_tx_fifo_alarm (event_id2) */ SDMA_REQ_ATA_RX = 4, /* ata_rcv_fifo_alarm (event_id2) */ SDMA_REQ_SIM = 5, /* */ @@ -223,5 +223,6 @@ struct channel_descriptor *cd_p, struct buffer_descriptor *base_bd_p); void sdma_channel_close(unsigned int channel); +bool sdma_channel_is_error(unsigned int channel); #endif /* SDMA_IMX31_H */ Index: firmware/target/arm/ata-pp5020.c =================================================================== --- firmware/target/arm/ata-pp5020.c (revision 20280) +++ firmware/target/arm/ata-pp5020.c (working copy) @@ -24,6 +24,9 @@ #include #include "system.h" #include "ata-target.h" +#ifdef HAVE_ATA_DMA +#include "ata.h" +#endif void ata_reset() { @@ -44,6 +47,13 @@ void ata_device_init() { +#ifdef HAVE_ATA_DMA + /* TODO: Is it necessary to wait for ready here? */ + IDE_DMA_CONTROL |= 2; + IDE_DMA_CONTROL &= ~1; + IDE0_CFG &= ~0x8010; + IDE0_CFG |= 0x20; +#else /* From ipod-ide.c:ipod_ide_register() */ IDE0_CFG |= (1<<5); #ifdef IPOD_NANO @@ -51,7 +61,174 @@ #else IDE0_CFG &=~(0x10000000); /* cpu < 65MHz */ #endif +#endif IDE0_PRI_TIMING0 = 0x10; IDE0_PRI_TIMING1 = 0x80002150; } + +/* These are PIO timings for 80 Mhz. At 24 Mhz, + the first value is 0 but the rest are the same. + They go in IDE0_PRI_TIMING0. + + Rockbox used 0x10, and test_disk shows that leads to faster PIO. + If 0x10 is incorrect, these timings may be needed with some devices. +static const unsigned long pio80mhz[] = { + 0xC293, 0x43A2, 0x11A1, 0x7232, 0x3131 +}; +*/ + +#ifdef HAVE_ATA_DMA +/* Timings for multi-word and ultra DMA modes. + These go in IDE0_PRI_TIMING1 + */ +static const unsigned long tm_mwdma[] = { + 0xF9F92, 0x56562, 0x45451 +}; + +static const unsigned long tm_udma[] = { + 0x800037C1, 0x80003491, 0x80003371, +#if ATA_MAX_UDMA > 2 + 0x80003271, 0x80003071 +#endif +}; + +#if ATA_MAX_UDMA > 2 +static bool dma_boosted = false; +static bool dma_needs_boost; +#endif + +/* This function sets up registers for 80 Mhz. + Ultra DMA mode 2 works at 30 Mhz. + */ +void ata_dma_set_mode(unsigned char mode) { + int modeidx; + + (*(volatile unsigned long *)(0x600060C4)) = 0xC0000000; /* 80 Mhz */ + IDE0_CFG &= ~0x10000000; + + modeidx = mode & 7; + mode &= 0xF8; + if (mode == 0x40 && modeidx <= ATA_MAX_UDMA) { + IDE0_PRI_TIMING1 = tm_udma[modeidx]; +#if ATA_MAX_UDMA > 2 + if (modeidx > 2) + dma_needs_boost = true; + else + dma_needs_boost = false; +#endif + } else if (mode == 0x20 && modeidx <= ATA_MAX_MWDMA) + IDE0_PRI_TIMING1 = tm_mwdma[modeidx]; + + IDE0_CFG |= 0x20000000; /* >= 50 Mhz */ +} + +#define IDE_CFG_INTRQ 8 +#define IDE_DMA_CONTROL_READ 8 + +/* This waits for an ATA interrupt using polling. + In ATA_CONTROL, CONTROL_nIEN must be cleared. + */ +STATICIRAM ICODE_ATTR int ata_wait_intrq(void) +{ + long timeout = current_tick + HZ*10; + + do + { + if (IDE0_CFG & IDE_CFG_INTRQ) + return 1; + last_disk_activity = current_tick; + yield(); + } while (TIME_BEFORE(current_tick, timeout)); + + return 0; /* timeout */ +} + +/* This function checks if parameters are appropriate for DMA, + and if they are, it sets up for DMA. + + If return value is false, caller may use PIO for this transfer. + + If return value is true, caller must issue a DMA ATA command + and then call ata_dma_finish(). + */ +bool ata_dma_setup(void *addr, unsigned long bytes, bool write) { + if (bytes <= 512 || ((unsigned long)addr & 3)) + return false; + +#if ATA_MAX_UDMA > 2 + if (dma_needs_boost && !dma_boosted) { + cpu_boost(true); + dma_boosted = true; + } +#endif + + if (write) { + /* If unflushed, old data may be written to disk */ + cpucache_flush(); + } + else { + /* Invalidate cache because new data may be present in RAM */ + cpucache_invalidate(); + /* TODO: Alignment to cache line boundaries should be performed + pre/post DMA */ + } + + /* Clear pending interrupts so ata_dma_finish() can wait for an + interrupt from this transfer + */ + IDE0_CFG |= IDE_CFG_INTRQ; + + IDE_DMA_CONTROL |= 2; + IDE_DMA_LENGTH = bytes - 4; + +#ifndef BOOTLOADER + if ((unsigned long)addr < DRAM_START) + /* Rockbox remaps DRAM to start at 0 */ + IDE_DMA_ADDR = (unsigned long)addr + DRAM_START; + else +#endif + IDE_DMA_ADDR = (unsigned long)addr; + + if (write) + IDE_DMA_CONTROL &= ~IDE_DMA_CONTROL_READ; + else + IDE_DMA_CONTROL |= IDE_DMA_CONTROL_READ; + + IDE0_CFG |= 0x8000; + + return true; +} + +/* This function waits for a DMA transfer to end. + It must be called to finish what ata_dma_setup started. + + Return value is true if DMA completed before the timeout, and false + if a timeout happened. + */ +bool ata_dma_finish(void) { + bool res; + + /* It may be okay to put this at the end of setup */ + IDE_DMA_CONTROL |= 1; + + /* Wait for end of transfer. + Reading standard ATA status while DMA is in progress causes + failures and hangs. Because of that, another wait is used. + */ + res = ata_wait_intrq(); + + IDE0_CFG &= ~0x8000; + IDE_DMA_CONTROL &= ~0x80000001; + +#if ATA_MAX_UDMA > 2 + if (dma_boosted) { + cpu_boost(false); + dma_boosted = false; + } +#endif + + return res; +} + +#endif /* HAVE_ATA_DMA */ Index: firmware/drivers/ata.c =================================================================== --- firmware/drivers/ata.c (revision 20280) +++ firmware/drivers/ata.c (working copy) @@ -60,6 +60,12 @@ #define CMD_SLEEP 0xE6 #define CMD_SET_FEATURES 0xEF #define CMD_SECURITY_FREEZE_LOCK 0xF5 +#ifdef HAVE_ATA_DMA +#define CMD_READ_DMA 0xC8 +#define CMD_READ_DMA_EXT 0x25 +#define CMD_WRITE_DMA 0xCA +#define CMD_WRITE_DMA_EXT 0x35 +#endif /* Should all be < 0x100 (which are reserved for control messages) */ #define Q_SLEEP 0 @@ -169,7 +175,12 @@ static bool initialized = false; static long last_user_activity = -1; +#ifdef HAVE_ATA_DMA +/* Needed to allow updating while waiting for DMA to complete */ +long last_disk_activity = -1; +#else static long last_disk_activity = -1; +#endif static unsigned long total_sectors; static int multisectors; /* number of supported multisectors */ @@ -188,6 +199,10 @@ static int phys_sector_mult = 1; #endif +#ifdef HAVE_ATA_DMA +static int dma_mode = 0; +#endif + static int ata_power_on(void); static int perform_soft_reset(void); static int set_multiple_mode(int sectors); @@ -240,7 +255,8 @@ { if (!wait_for_bsy()) return 0; - return (ATA_ALT_STATUS & (STATUS_RDY|STATUS_DRQ)) == STATUS_RDY; + /* See FS#9721 */ + return (ATA_ALT_STATUS & (STATUS_BSY|STATUS_RDY|STATUS_DF|STATUS_DRQ|STATUS_ERR)) == STATUS_RDY; } #if (CONFIG_LED == LED_REAL) @@ -308,6 +324,9 @@ int count; void* buf; long spinup_start; +#ifdef HAVE_ATA_DMA + bool usedma = false; +#endif #ifndef MAX_PHYS_SECTOR_SIZE #ifdef HAVE_MULTIVOLUME @@ -358,6 +377,12 @@ ret = 0; last_disk_activity = current_tick; +#ifdef HAVE_ATA_DMA + /* If DMA is supported and parameters are ok for DMA, use it */ + if (dma_mode && ata_dma_setup(inbuf, incount * SECTOR_SIZE, false)) + usedma = true; +#endif + #ifdef HAVE_LBA48 if (lba48) { @@ -370,7 +395,11 @@ SET_REG(ATA_HCYL, 0); /* 47:40 */ SET_REG(ATA_HCYL, (start >> 16) & 0xff); /* 23:16 */ SET_REG(ATA_SELECT, SELECT_LBA | ata_device); +#ifdef HAVE_ATA_DMA + SET_REG(ATA_COMMAND, usedma ? CMD_READ_DMA_EXT : CMD_READ_MULTIPLE_EXT); +#else SET_REG(ATA_COMMAND, CMD_READ_MULTIPLE_EXT); +#endif } else #endif @@ -380,7 +409,11 @@ SET_REG(ATA_LCYL, (start >> 8) & 0xff); SET_REG(ATA_HCYL, (start >> 16) & 0xff); SET_REG(ATA_SELECT, ((start >> 24) & 0xf) | SELECT_LBA | ata_device); +#ifdef HAVE_ATA_DMA + SET_REG(ATA_COMMAND, usedma ? CMD_READ_DMA : CMD_READ_MULTIPLE); +#else SET_REG(ATA_COMMAND, CMD_READ_MULTIPLE); +#endif } /* wait at least 400ns between writing command and reading status */ @@ -390,6 +423,25 @@ __asm__ volatile ("nop"); __asm__ volatile ("nop"); +#ifdef HAVE_ATA_DMA + if (usedma) { + if (!ata_dma_finish()) + ret = -7; + + if (ret != 0) { + perform_soft_reset(); + goto retry; + } + + if (spinup) { + spinup_time = current_tick - spinup_start; + spinup = false; + sleeping = false; + poweroff = false; + } + } + else +#endif /* HAVE_ATA_DMA */ while (count) { int sectors; int wordcount; @@ -515,6 +567,9 @@ int i; int ret = 0; long spinup_start; +#ifdef HAVE_ATA_DMA + bool usedma = false; +#endif #ifndef MAX_PHYS_SECTOR_SIZE #ifdef HAVE_MULTIVOLUME @@ -554,6 +609,12 @@ goto error; } +#ifdef HAVE_ATA_DMA + /* If DMA is supported and parameters are ok for DMA, use it */ + if (dma_mode && ata_dma_setup((void *)buf, count * SECTOR_SIZE, true)) + usedma = true; +#endif + #ifdef HAVE_LBA48 if (lba48) { @@ -566,7 +627,11 @@ SET_REG(ATA_HCYL, 0); /* 47:40 */ SET_REG(ATA_HCYL, (start >> 16) & 0xff); /* 23:16 */ SET_REG(ATA_SELECT, SELECT_LBA | ata_device); +#ifdef HAVE_ATA_DMA + SET_REG(ATA_COMMAND, usedma ? CMD_WRITE_DMA_EXT : CMD_WRITE_SECTORS_EXT); +#else SET_REG(ATA_COMMAND, CMD_WRITE_SECTORS_EXT); +#endif } else #endif @@ -576,9 +641,26 @@ SET_REG(ATA_LCYL, (start >> 8) & 0xff); SET_REG(ATA_HCYL, (start >> 16) & 0xff); SET_REG(ATA_SELECT, ((start >> 24) & 0xf) | SELECT_LBA | ata_device); +#ifdef HAVE_ATA_DMA + SET_REG(ATA_COMMAND, usedma ? CMD_WRITE_DMA : CMD_WRITE_SECTORS); +#else SET_REG(ATA_COMMAND, CMD_WRITE_SECTORS); +#endif } +#ifdef HAVE_ATA_DMA + if (usedma) { + if (!ata_dma_finish()) + ret = -7; + else if (spinup) { + spinup_time = current_tick - spinup_start; + spinup = false; + sleeping = false; + poweroff = false; + } + } + else +#endif /* HAVE_ATA_DMA */ for (i=0; i= 5us */ +#ifdef HAVE_ATA_DMA + /* DMA requires INTRQ be enabled */ + SET_REG(ATA_CONTROL, 0); +#else SET_REG(ATA_CONTROL, CONTROL_nIEN); +#endif sleep(1); /* >2ms */ /* This little sucker can take up to 30 seconds */ @@ -1179,6 +1266,22 @@ return 0; } +#ifdef HAVE_ATA_DMA +int get_best_mode(unsigned short identword, int max, int modetype) +{ + unsigned short testbit = 1u << max; + + while (1) { + if (identword & testbit) + return max | modetype; + testbit >>= 1; + if (!testbit) + return 0; + max--; + } +} +#endif + static int set_features(void) { static struct { @@ -1191,6 +1294,9 @@ { 83, 3, 0x05, 0x80 }, /* adv. power management: lowest w/o standby */ { 83, 9, 0x42, 0x80 }, /* acoustic management: lowest noise */ { 82, 6, 0xaa, 0 }, /* enable read look-ahead */ +#ifdef HAVE_ATA_DMA + { 0, 0, 0x03, 0 }, /* DMA mode */ +#endif }; int i; int pio_mode = 2; @@ -1204,7 +1310,24 @@ /* Update the table: set highest supported pio mode that we also support */ features[0].parameter = 8 + pio_mode; + +#ifdef HAVE_ATA_DMA + if (identify_info[53] & (1<<2)) + /* Ultra DMA mode info present, find a mode */ + dma_mode = get_best_mode(identify_info[88], ATA_MAX_UDMA, 0x40); + if (!dma_mode) { + /* No UDMA mode found, try to find a multi-word DMA mode */ + dma_mode = get_best_mode(identify_info[63], ATA_MAX_MWDMA, 0x20); + features[4].id_word = 63; + } + else + features[4].id_word = 88; + + features[4].id_bit = dma_mode & 7; + features[4].parameter = dma_mode; +#endif /* HAVE_ATA_DMA */ + SET_REG(ATA_SELECT, ata_device); if (!wait_for_rdy()) { @@ -1237,6 +1360,10 @@ ata_set_pio_timings(pio_mode); #endif +#ifdef HAVE_ATA_DMA + ata_dma_set_mode(dma_mode); +#endif + return 0; } @@ -1305,6 +1432,11 @@ sleep(HZ/4); /* allow voltage to build up */ } +#ifdef HAVE_ATA_DMA + /* DMA requires INTRQ be enabled */ + SET_REG(ATA_CONTROL, 0); +#endif + /* first try, hard reset at cold start only */ rc = init_and_check(coldstart); @@ -1450,3 +1582,11 @@ info->revision=revision; } #endif + +#ifdef HAVE_ATA_DMA +/* Returns last DMA mode as set by set_features() */ +int ata_get_dma_mode(void) +{ + return dma_mode; +} +#endif