| .. | .. |
|---|
| 1 | 1 | #ifndef _HFI1_SDMA_H |
|---|
| 2 | 2 | #define _HFI1_SDMA_H |
|---|
| 3 | 3 | /* |
|---|
| 4 | | - * Copyright(c) 2015, 2016 Intel Corporation. |
|---|
| 4 | + * Copyright(c) 2015 - 2018 Intel Corporation. |
|---|
| 5 | 5 | * |
|---|
| 6 | 6 | * This file is provided under a dual BSD/GPLv2 license. When using or |
|---|
| 7 | 7 | * redistributing this file, you may do so under either license. |
|---|
| .. | .. |
|---|
| 61 | 61 | #define MAX_DESC 64 |
|---|
| 62 | 62 | /* Hardware limit for SDMA packet size */ |
|---|
| 63 | 63 | #define MAX_SDMA_PKT_SIZE ((16 * 1024) - 1) |
|---|
| 64 | | - |
|---|
| 65 | | -#define SDMA_TXREQ_S_OK 0 |
|---|
| 66 | | -#define SDMA_TXREQ_S_SENDERROR 1 |
|---|
| 67 | | -#define SDMA_TXREQ_S_ABORTED 2 |
|---|
| 68 | | -#define SDMA_TXREQ_S_SHUTDOWN 3 |
|---|
| 69 | | - |
|---|
| 70 | | -/* flags bits */ |
|---|
| 71 | | -#define SDMA_TXREQ_F_URGENT 0x0001 |
|---|
| 72 | | -#define SDMA_TXREQ_F_AHG_COPY 0x0002 |
|---|
| 73 | | -#define SDMA_TXREQ_F_USE_AHG 0x0004 |
|---|
| 74 | 64 | |
|---|
| 75 | 65 | #define SDMA_MAP_NONE 0 |
|---|
| 76 | 66 | #define SDMA_MAP_SINGLE 1 |
|---|
| .. | .. |
|---|
| 392 | 382 | u64 progress_int_cnt; |
|---|
| 393 | 383 | |
|---|
| 394 | 384 | /* private: */ |
|---|
| 385 | + seqlock_t waitlock; |
|---|
| 395 | 386 | struct list_head dmawait; |
|---|
| 396 | 387 | |
|---|
| 397 | 388 | /* CONFIG SDMA for now, just blindly duplicate */ |
|---|
| .. | .. |
|---|
| 415 | 406 | struct list_head flushlist; |
|---|
| 416 | 407 | struct cpumask cpu_mask; |
|---|
| 417 | 408 | struct kobject kobj; |
|---|
| 409 | + u32 msix_intr; |
|---|
| 418 | 410 | }; |
|---|
| 419 | 411 | |
|---|
| 420 | 412 | int sdma_init(struct hfi1_devdata *dd, u8 port); |
|---|
| .. | .. |
|---|
| 644 | 636 | struct sdma_txreq *tx, |
|---|
| 645 | 637 | int type, |
|---|
| 646 | 638 | dma_addr_t addr, |
|---|
| 647 | | - size_t len) |
|---|
| 639 | + size_t len, |
|---|
| 640 | + void *pinning_ctx, |
|---|
| 641 | + void (*ctx_get)(void *), |
|---|
| 642 | + void (*ctx_put)(void *)) |
|---|
| 648 | 643 | { |
|---|
| 649 | 644 | struct sdma_desc *desc = &tx->descp[tx->num_desc]; |
|---|
| 650 | 645 | |
|---|
| .. | .. |
|---|
| 661 | 656 | << SDMA_DESC0_PHY_ADDR_SHIFT) | |
|---|
| 662 | 657 | (((u64)len & SDMA_DESC0_BYTE_COUNT_MASK) |
|---|
| 663 | 658 | << SDMA_DESC0_BYTE_COUNT_SHIFT); |
|---|
| 659 | + |
|---|
| 660 | + desc->pinning_ctx = pinning_ctx; |
|---|
| 661 | + desc->ctx_put = ctx_put; |
|---|
| 662 | + if (pinning_ctx && ctx_get) |
|---|
| 663 | + ctx_get(pinning_ctx); |
|---|
| 664 | 664 | } |
|---|
| 665 | 665 | |
|---|
| 666 | 666 | /* helper to extend txreq */ |
|---|
| .. | .. |
|---|
| 680 | 680 | static inline void _sdma_close_tx(struct hfi1_devdata *dd, |
|---|
| 681 | 681 | struct sdma_txreq *tx) |
|---|
| 682 | 682 | { |
|---|
| 683 | | - tx->descp[tx->num_desc].qw[0] |= |
|---|
| 684 | | - SDMA_DESC0_LAST_DESC_FLAG; |
|---|
| 685 | | - tx->descp[tx->num_desc].qw[1] |= |
|---|
| 686 | | - dd->default_desc1; |
|---|
| 683 | + u16 last_desc = tx->num_desc - 1; |
|---|
| 684 | + |
|---|
| 685 | + tx->descp[last_desc].qw[0] |= SDMA_DESC0_LAST_DESC_FLAG; |
|---|
| 686 | + tx->descp[last_desc].qw[1] |= dd->default_desc1; |
|---|
| 687 | 687 | if (tx->flags & SDMA_TXREQ_F_URGENT) |
|---|
| 688 | | - tx->descp[tx->num_desc].qw[1] |= |
|---|
| 689 | | - (SDMA_DESC1_HEAD_TO_HOST_FLAG | |
|---|
| 690 | | - SDMA_DESC1_INT_REQ_FLAG); |
|---|
| 688 | + tx->descp[last_desc].qw[1] |= (SDMA_DESC1_HEAD_TO_HOST_FLAG | |
|---|
| 689 | + SDMA_DESC1_INT_REQ_FLAG); |
|---|
| 691 | 690 | } |
|---|
| 692 | 691 | |
|---|
| 693 | 692 | static inline int _sdma_txadd_daddr( |
|---|
| .. | .. |
|---|
| 695 | 694 | int type, |
|---|
| 696 | 695 | struct sdma_txreq *tx, |
|---|
| 697 | 696 | dma_addr_t addr, |
|---|
| 698 | | - u16 len) |
|---|
| 697 | + u16 len, |
|---|
| 698 | + void *pinning_ctx, |
|---|
| 699 | + void (*ctx_get)(void *), |
|---|
| 700 | + void (*ctx_put)(void *)) |
|---|
| 699 | 701 | { |
|---|
| 700 | 702 | int rval = 0; |
|---|
| 701 | 703 | |
|---|
| 702 | 704 | make_tx_sdma_desc( |
|---|
| 703 | 705 | tx, |
|---|
| 704 | 706 | type, |
|---|
| 705 | | - addr, len); |
|---|
| 707 | + addr, len, |
|---|
| 708 | + pinning_ctx, ctx_get, ctx_put); |
|---|
| 706 | 709 | WARN_ON(len > tx->tlen); |
|---|
| 710 | + tx->num_desc++; |
|---|
| 707 | 711 | tx->tlen -= len; |
|---|
| 708 | 712 | /* special cases for last */ |
|---|
| 709 | 713 | if (!tx->tlen) { |
|---|
| .. | .. |
|---|
| 715 | 719 | _sdma_close_tx(dd, tx); |
|---|
| 716 | 720 | } |
|---|
| 717 | 721 | } |
|---|
| 718 | | - tx->num_desc++; |
|---|
| 719 | 722 | return rval; |
|---|
| 720 | 723 | } |
|---|
| 721 | 724 | |
|---|
| .. | .. |
|---|
| 726 | 729 | * @page: page to map |
|---|
| 727 | 730 | * @offset: offset within the page |
|---|
| 728 | 731 | * @len: length in bytes |
|---|
| 732 | + * @pinning_ctx: context to be stored on struct sdma_desc .pinning_ctx. Not |
|---|
| 733 | + * added if coalesce buffer is used. E.g. pointer to pinned-page |
|---|
| 734 | + * cache entry for the sdma_desc. |
|---|
| 735 | + * @ctx_get: optional function to take reference to @pinning_ctx. Not called if |
|---|
| 736 | + * @pinning_ctx is NULL. |
|---|
| 737 | + * @ctx_put: optional function to release reference to @pinning_ctx after |
|---|
| 738 | + * sdma_desc completes. May be called in interrupt context so must |
|---|
| 739 | + * not sleep. Not called if @pinning_ctx is NULL. |
|---|
| 729 | 740 | * |
|---|
| 730 | 741 | * This is used to add a page/offset/length descriptor. |
|---|
| 731 | 742 | * |
|---|
| .. | .. |
|---|
| 740 | 751 | struct sdma_txreq *tx, |
|---|
| 741 | 752 | struct page *page, |
|---|
| 742 | 753 | unsigned long offset, |
|---|
| 743 | | - u16 len) |
|---|
| 754 | + u16 len, |
|---|
| 755 | + void *pinning_ctx, |
|---|
| 756 | + void (*ctx_get)(void *), |
|---|
| 757 | + void (*ctx_put)(void *)) |
|---|
| 744 | 758 | { |
|---|
| 745 | 759 | dma_addr_t addr; |
|---|
| 746 | 760 | int rval; |
|---|
| .. | .. |
|---|
| 764 | 778 | return -ENOSPC; |
|---|
| 765 | 779 | } |
|---|
| 766 | 780 | |
|---|
| 767 | | - return _sdma_txadd_daddr( |
|---|
| 768 | | - dd, SDMA_MAP_PAGE, tx, addr, len); |
|---|
| 781 | + return _sdma_txadd_daddr(dd, SDMA_MAP_PAGE, tx, addr, len, |
|---|
| 782 | + pinning_ctx, ctx_get, ctx_put); |
|---|
| 769 | 783 | } |
|---|
| 770 | 784 | |
|---|
| 771 | 785 | /** |
|---|
| .. | .. |
|---|
| 799 | 813 | return rval; |
|---|
| 800 | 814 | } |
|---|
| 801 | 815 | |
|---|
| 802 | | - return _sdma_txadd_daddr(dd, SDMA_MAP_NONE, tx, addr, len); |
|---|
| 816 | + return _sdma_txadd_daddr(dd, SDMA_MAP_NONE, tx, addr, len, |
|---|
| 817 | + NULL, NULL, NULL); |
|---|
| 803 | 818 | } |
|---|
| 804 | 819 | |
|---|
| 805 | 820 | /** |
|---|
| .. | .. |
|---|
| 845 | 860 | return -ENOSPC; |
|---|
| 846 | 861 | } |
|---|
| 847 | 862 | |
|---|
| 848 | | - return _sdma_txadd_daddr( |
|---|
| 849 | | - dd, SDMA_MAP_SINGLE, tx, addr, len); |
|---|
| 863 | + return _sdma_txadd_daddr(dd, SDMA_MAP_SINGLE, tx, addr, len, |
|---|
| 864 | + NULL, NULL, NULL); |
|---|
| 850 | 865 | } |
|---|
| 851 | 866 | |
|---|
| 852 | | -struct iowait; |
|---|
| 867 | +struct iowait_work; |
|---|
| 853 | 868 | |
|---|
| 854 | 869 | int sdma_send_txreq(struct sdma_engine *sde, |
|---|
| 855 | | - struct iowait *wait, |
|---|
| 870 | + struct iowait_work *wait, |
|---|
| 856 | 871 | struct sdma_txreq *tx, |
|---|
| 857 | 872 | bool pkts_sent); |
|---|
| 858 | 873 | int sdma_send_txlist(struct sdma_engine *sde, |
|---|
| 859 | | - struct iowait *wait, |
|---|
| 874 | + struct iowait_work *wait, |
|---|
| 860 | 875 | struct list_head *tx_list, |
|---|
| 861 | | - u32 *count); |
|---|
| 876 | + u16 *count_out); |
|---|
| 862 | 877 | |
|---|
| 863 | 878 | int sdma_ahg_alloc(struct sdma_engine *sde); |
|---|
| 864 | 879 | void sdma_ahg_free(struct sdma_engine *sde, int ahg_index); |
|---|
| .. | .. |
|---|
| 1010 | 1025 | */ |
|---|
| 1011 | 1026 | struct sdma_map_elem { |
|---|
| 1012 | 1027 | u32 mask; |
|---|
| 1013 | | - struct sdma_engine *sde[0]; |
|---|
| 1028 | + struct sdma_engine *sde[]; |
|---|
| 1014 | 1029 | }; |
|---|
| 1015 | 1030 | |
|---|
| 1016 | 1031 | /** |
|---|
| .. | .. |
|---|
| 1032 | 1047 | u32 mask; |
|---|
| 1033 | 1048 | u8 actual_vls; |
|---|
| 1034 | 1049 | u8 vls; |
|---|
| 1035 | | - struct sdma_map_elem *map[0]; |
|---|
| 1050 | + struct sdma_map_elem *map[]; |
|---|
| 1036 | 1051 | }; |
|---|
| 1037 | 1052 | |
|---|
| 1038 | 1053 | int sdma_map_init( |
|---|
| .. | .. |
|---|
| 1097 | 1112 | extern uint mod_num_sdma; |
|---|
| 1098 | 1113 | |
|---|
| 1099 | 1114 | void sdma_update_lmc(struct hfi1_devdata *dd, u64 mask, u32 lid); |
|---|
| 1100 | | - |
|---|
| 1101 | 1115 | #endif |
|---|