~ljy/RK3588_XEN.git

2024-11-15 a46a1ad097419aeea7350987dd95230f50d90392

commit \| author \| age
a07526	1	// SPDX-License-Identifier: GPL-2.0-only
H	2	/*******************************************************************************
	3	This is the driver for the ST MAC 10/100/1000 on-chip Ethernet controllers.
	4	ST Ethernet IPs are built around a Synopsys IP Core.
	5
	6	Copyright(C) 2007-2011 STMicroelectronics Ltd
	7
	8
	9	Author: Giuseppe Cavallaro <peppe.cavallaro@st.com>
	10
	11	Documentation available at:
	12	http://www.stlinux.com
	13	Support available at:
	14	https://bugzilla.stlinux.com/
	15	*******************************************************************************/
	16
	17	#include <linux/clk.h>
	18	#include <linux/kernel.h>
	19	#include <linux/interrupt.h>
	20	#include <linux/ip.h>
	21	#include <linux/tcp.h>
	22	#include <linux/skbuff.h>
	23	#include <linux/ethtool.h>
	24	#include <linux/if_ether.h>
	25	#include <linux/crc32.h>
	26	#include <linux/mii.h>
	27	#include <linux/if.h>
	28	#include <linux/if_vlan.h>
	29	#include <linux/dma-mapping.h>
	30	#include <linux/slab.h>
	31	#include <linux/pm_runtime.h>
	32	#include <linux/prefetch.h>
	33	#include <linux/pinctrl/consumer.h>
	34	#ifdef CONFIG_DEBUG_FS
	35	#include <linux/debugfs.h>
	36	#include <linux/seq_file.h>
	37	#endif /* CONFIG_DEBUG_FS */
	38	#include <linux/net_tstamp.h>
	39	#include <linux/phylink.h>
	40	#include <linux/udp.h>
	41	#include <net/pkt_cls.h>
	42	#include "stmmac_ptp.h"
	43	#include "stmmac.h"
	44	#include <linux/reset.h>
	45	#include <linux/of_mdio.h>
	46	#include "dwmac1000.h"
	47	#include "dwxgmac2.h"
	48	#include "hwif.h"
	49
	50	/* As long as the interface is active, we keep the timestamping counter enabled
	51	* with fine resolution and binary rollover. This avoid non-monotonic behavior
	52	* (clock jumps) when changing timestamping settings at runtime.
	53	*/
	54	#define STMMAC_HWTS_ACTIVE (PTP_TCR_TSENA \| PTP_TCR_TSCFUPDT \| \
	55	PTP_TCR_TSCTRLSSR)
	56
	57	#define STMMAC_ALIGN(x) ALIGN(ALIGN(x, SMP_CACHE_BYTES), 16)
	58	#define TSO_MAX_BUFF_SIZE (SZ_16K - 1)
	59
	60	/* Module parameters */
	61	#define TX_TIMEO 5000
	62	static int watchdog = TX_TIMEO;
	63	module_param(watchdog, int, 0644);
	64	MODULE_PARM_DESC(watchdog, "Transmit timeout in milliseconds (default 5s)");
	65
	66	static int debug = -1;
	67	module_param(debug, int, 0644);
	68	MODULE_PARM_DESC(debug, "Message Level (-1: default, 0: no output, 16: all)");
	69
	70	static int phyaddr = -1;
	71	module_param(phyaddr, int, 0444);
	72	MODULE_PARM_DESC(phyaddr, "Physical device address");
	73
	74	#define STMMAC_TX_THRESH(x) ((x)->dma_tx_size / 4)
	75	#define STMMAC_RX_THRESH(x) ((x)->dma_rx_size / 4)
	76
	77	static int flow_ctrl = FLOW_AUTO;
	78	module_param(flow_ctrl, int, 0644);
	79	MODULE_PARM_DESC(flow_ctrl, "Flow control ability [on/off]");
	80
	81	static int pause = PAUSE_TIME;
	82	module_param(pause, int, 0644);
	83	MODULE_PARM_DESC(pause, "Flow Control Pause Time");
	84
	85	#define TC_DEFAULT 64
	86	static int tc = TC_DEFAULT;
	87	module_param(tc, int, 0644);
	88	MODULE_PARM_DESC(tc, "DMA threshold control value");
	89
	90	#define DEFAULT_BUFSIZE 1536
	91	static int buf_sz = DEFAULT_BUFSIZE;
	92	module_param(buf_sz, int, 0644);
	93	MODULE_PARM_DESC(buf_sz, "DMA buffer size");
	94
	95	#define STMMAC_RX_COPYBREAK 256
	96
	97	static const u32 default_msg_level = (NETIF_MSG_DRV \| NETIF_MSG_PROBE \|
	98	NETIF_MSG_LINK \| NETIF_MSG_IFUP \|
	99	NETIF_MSG_IFDOWN \| NETIF_MSG_TIMER);
	100
	101	#define STMMAC_DEFAULT_LPI_TIMER 1000
	102	static int eee_timer = STMMAC_DEFAULT_LPI_TIMER;
	103	module_param(eee_timer, int, 0644);
	104	MODULE_PARM_DESC(eee_timer, "LPI tx expiration time in msec");
	105	#define STMMAC_LPI_T(x) (jiffies + usecs_to_jiffies(x))
	106
	107	/* By default the driver will use the ring mode to manage tx and rx descriptors,
	108	* but allow user to force to use the chain instead of the ring
	109	*/
	110	static unsigned int chain_mode;
	111	module_param(chain_mode, int, 0444);
	112	MODULE_PARM_DESC(chain_mode, "To use chain instead of ring mode");
	113
	114	static irqreturn_t stmmac_interrupt(int irq, void *dev_id);
	115
	116	#ifdef CONFIG_DEBUG_FS
	117	static const struct net_device_ops stmmac_netdev_ops;
	118	static void stmmac_init_fs(struct net_device *dev);
	119	static void stmmac_exit_fs(struct net_device *dev);
	120	#endif
	121
	122	#define STMMAC_COAL_TIMER(x) (jiffies + usecs_to_jiffies(x))
	123
	124	#define LED_FIX 1
	125	#ifdef LED_FIX
	126	#define RTL_8201F_PHY_ID 0x001cc816
	127	#define RTL_8211E_PHY_ID 0x001cc915
	128	#define RTL_8211F_PHY_ID 0x001cc916
	129	#define DP_83848_PHY_ID 0x20005c90
	130	#endif
	131
	132	int stmmac_bus_clks_config(struct stmmac_priv *priv, bool enabled)
	133	{
	134	int ret = 0;
	135
	136	if (enabled) {
	137	ret = clk_prepare_enable(priv->plat->stmmac_clk);
	138	if (ret)
	139	return ret;
	140	ret = clk_prepare_enable(priv->plat->pclk);
	141	if (ret) {
	142	clk_disable_unprepare(priv->plat->stmmac_clk);
	143	return ret;
	144	}
	145	} else {
	146	clk_disable_unprepare(priv->plat->stmmac_clk);
	147	clk_disable_unprepare(priv->plat->pclk);
	148	}
	149
	150	return ret;
	151	}
	152	EXPORT_SYMBOL_GPL(stmmac_bus_clks_config);
	153
	154	/**
	155	* stmmac_verify_args - verify the driver parameters.
	156	* Description: it checks the driver parameters and set a default in case of
	157	* errors.
	158	*/
	159	static void stmmac_verify_args(void)
	160	{
	161	if (unlikely(watchdog < 0))
	162	watchdog = TX_TIMEO;
	163	if (unlikely((buf_sz < DEFAULT_BUFSIZE) \|\| (buf_sz > BUF_SIZE_16KiB)))
	164	buf_sz = DEFAULT_BUFSIZE;
	165	if (unlikely(flow_ctrl > 1))
	166	flow_ctrl = FLOW_AUTO;
	167	else if (likely(flow_ctrl < 0))
	168	flow_ctrl = FLOW_OFF;
	169	if (unlikely((pause < 0) \|\| (pause > 0xffff)))
	170	pause = PAUSE_TIME;
	171	if (eee_timer < 0)
	172	eee_timer = STMMAC_DEFAULT_LPI_TIMER;
	173	}
	174
	175	/**
	176	* stmmac_disable_all_queues - Disable all queues
	177	* @priv: driver private structure
	178	*/
	179	static void stmmac_disable_all_queues(struct stmmac_priv *priv)
	180	{
	181	u32 rx_queues_cnt = priv->plat->rx_queues_to_use;
	182	u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
	183	u32 maxq = max(rx_queues_cnt, tx_queues_cnt);
	184	u32 queue;
	185
	186	for (queue = 0; queue < maxq; queue++) {
	187	struct stmmac_channel *ch = &priv->channel[queue];
	188
	189	if (queue < rx_queues_cnt)
	190	napi_disable(&ch->rx_napi);
	191	if (queue < tx_queues_cnt)
	192	napi_disable(&ch->tx_napi);
	193	}
	194	}
	195
	196	/**
	197	* stmmac_enable_all_queues - Enable all queues
	198	* @priv: driver private structure
	199	*/
	200	static void stmmac_enable_all_queues(struct stmmac_priv *priv)
	201	{
	202	u32 rx_queues_cnt = priv->plat->rx_queues_to_use;
	203	u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
	204	u32 maxq = max(rx_queues_cnt, tx_queues_cnt);
	205	u32 queue;
	206
	207	for (queue = 0; queue < maxq; queue++) {
	208	struct stmmac_channel *ch = &priv->channel[queue];
	209
	210	if (queue < rx_queues_cnt)
	211	napi_enable(&ch->rx_napi);
	212	if (queue < tx_queues_cnt)
	213	napi_enable(&ch->tx_napi);
	214	}
	215	}
	216
	217	static void stmmac_service_event_schedule(struct stmmac_priv *priv)
	218	{
	219	if (!test_bit(STMMAC_DOWN, &priv->state) &&
	220	!test_and_set_bit(STMMAC_SERVICE_SCHED, &priv->state))
	221	queue_work(priv->wq, &priv->service_task);
	222	}
	223
	224	static void stmmac_global_err(struct stmmac_priv *priv)
	225	{
	226	netif_carrier_off(priv->dev);
	227	set_bit(STMMAC_RESET_REQUESTED, &priv->state);
	228	stmmac_service_event_schedule(priv);
	229	}
	230
	231	/**
	232	* stmmac_clk_csr_set - dynamically set the MDC clock
	233	* @priv: driver private structure
	234	* Description: this is to dynamically set the MDC clock according to the csr
	235	* clock input.
	236	* Note:
	237	* If a specific clk_csr value is passed from the platform
	238	* this means that the CSR Clock Range selection cannot be
	239	* changed at run-time and it is fixed (as reported in the driver
	240	* documentation). Viceversa the driver will try to set the MDC
	241	* clock dynamically according to the actual clock input.
	242	*/
	243	static void stmmac_clk_csr_set(struct stmmac_priv *priv)
	244	{
	245	u32 clk_rate;
	246
	247	clk_rate = clk_get_rate(priv->plat->pclk);
	248
	249	/* Platform provided default clk_csr would be assumed valid
	250	* for all other cases except for the below mentioned ones.
	251	* For values higher than the IEEE 802.3 specified frequency
	252	* we can not estimate the proper divider as it is not known
	253	* the frequency of clk_csr_i. So we do not change the default
	254	* divider.
	255	*/
	256	if (!(priv->clk_csr & MAC_CSR_H_FRQ_MASK)) {
	257	if (clk_rate < CSR_F_35M)
	258	priv->clk_csr = STMMAC_CSR_20_35M;
	259	else if ((clk_rate >= CSR_F_35M) && (clk_rate < CSR_F_60M))
	260	priv->clk_csr = STMMAC_CSR_35_60M;
	261	else if ((clk_rate >= CSR_F_60M) && (clk_rate < CSR_F_100M))
	262	priv->clk_csr = STMMAC_CSR_60_100M;
	263	else if ((clk_rate >= CSR_F_100M) && (clk_rate < CSR_F_150M))
	264	priv->clk_csr = STMMAC_CSR_100_150M;
	265	else if ((clk_rate >= CSR_F_150M) && (clk_rate < CSR_F_250M))
	266	priv->clk_csr = STMMAC_CSR_150_250M;
	267	else if ((clk_rate >= CSR_F_250M) && (clk_rate <= CSR_F_300M))
	268	priv->clk_csr = STMMAC_CSR_250_300M;
	269	}
	270
	271	if (priv->plat->has_sun8i) {
	272	if (clk_rate > 160000000)
	273	priv->clk_csr = 0x03;
	274	else if (clk_rate > 80000000)
	275	priv->clk_csr = 0x02;
	276	else if (clk_rate > 40000000)
	277	priv->clk_csr = 0x01;
	278	else
	279	priv->clk_csr = 0;
	280	}
	281
	282	if (priv->plat->has_xgmac) {
	283	if (clk_rate > 400000000)
	284	priv->clk_csr = 0x5;
	285	else if (clk_rate > 350000000)
	286	priv->clk_csr = 0x4;
	287	else if (clk_rate > 300000000)
	288	priv->clk_csr = 0x3;
	289	else if (clk_rate > 250000000)
	290	priv->clk_csr = 0x2;
	291	else if (clk_rate > 150000000)
	292	priv->clk_csr = 0x1;
	293	else
	294	priv->clk_csr = 0x0;
	295	}
	296	}
	297
	298	static void print_pkt(unsigned char *buf, int len)
	299	{
	300	pr_debug("len = %d byte, buf addr: 0x%p\n", len, buf);
	301	print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, buf, len);
	302	}
	303
	304	static inline u32 stmmac_tx_avail(struct stmmac_priv *priv, u32 queue)
	305	{
	306	struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
	307	u32 avail;
	308
	309	if (tx_q->dirty_tx > tx_q->cur_tx)
	310	avail = tx_q->dirty_tx - tx_q->cur_tx - 1;
	311	else
	312	avail = priv->dma_tx_size - tx_q->cur_tx + tx_q->dirty_tx - 1;
	313
	314	return avail;
	315	}
	316
	317	/**
	318	* stmmac_rx_dirty - Get RX queue dirty
	319	* @priv: driver private structure
	320	* @queue: RX queue index
	321	*/
	322	static inline u32 stmmac_rx_dirty(struct stmmac_priv *priv, u32 queue)
	323	{
	324	struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
	325	u32 dirty;
	326
	327	if (rx_q->dirty_rx <= rx_q->cur_rx)
	328	dirty = rx_q->cur_rx - rx_q->dirty_rx;
	329	else
	330	dirty = priv->dma_rx_size - rx_q->dirty_rx + rx_q->cur_rx;
	331
	332	return dirty;
	333	}
	334
	335	/**
	336	* stmmac_enable_eee_mode - check and enter in LPI mode
	337	* @priv: driver private structure
	338	* Description: this function is to verify and enter in LPI mode in case of
	339	* EEE.
	340	*/
	341	static void stmmac_enable_eee_mode(struct stmmac_priv *priv)
	342	{
	343	u32 tx_cnt = priv->plat->tx_queues_to_use;
	344	u32 queue;
	345
	346	/* check if all TX queues have the work finished */
	347	for (queue = 0; queue < tx_cnt; queue++) {
	348	struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
	349
	350	if (tx_q->dirty_tx != tx_q->cur_tx)
	351	return; /* still unfinished work */
	352	}
	353
	354	/* Check and enter in LPI mode */
	355	if (!priv->tx_path_in_lpi_mode)
	356	stmmac_set_eee_mode(priv, priv->hw,
	357	priv->plat->en_tx_lpi_clockgating);
	358	}
	359
	360	/**
	361	* stmmac_disable_eee_mode - disable and exit from LPI mode
	362	* @priv: driver private structure
	363	* Description: this function is to exit and disable EEE in case of
	364	* LPI state is true. This is called by the xmit.
	365	*/
	366	void stmmac_disable_eee_mode(struct stmmac_priv *priv)
	367	{
	368	stmmac_reset_eee_mode(priv, priv->hw);
	369	del_timer_sync(&priv->eee_ctrl_timer);
	370	priv->tx_path_in_lpi_mode = false;
	371	}
	372
	373	/**
	374	* stmmac_eee_ctrl_timer - EEE TX SW timer.
	375	* @t: timer_list struct containing private info
	376	* Description:
	377	* if there is no data transfer and if we are not in LPI state,
	378	* then MAC Transmitter can be moved to LPI state.
	379	*/
	380	static void stmmac_eee_ctrl_timer(struct timer_list *t)
	381	{
	382	struct stmmac_priv *priv = from_timer(priv, t, eee_ctrl_timer);
	383
	384	stmmac_enable_eee_mode(priv);
	385	mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer));
	386	}
	387
	388	/**
	389	* stmmac_eee_init - init EEE
	390	* @priv: driver private structure
	391	* Description:
	392	* if the GMAC supports the EEE (from the HW cap reg) and the phy device
	393	* can also manage EEE, this function enable the LPI state and start related
	394	* timer.
	395	*/
	396	bool stmmac_eee_init(struct stmmac_priv *priv)
	397	{
	398	int eee_tw_timer = priv->eee_tw_timer;
	399
	400	/* Using PCS we cannot dial with the phy registers at this stage
	401	* so we do not support extra feature like EEE.
	402	*/
	403	if (priv->hw->pcs == STMMAC_PCS_TBI \|\|
	404	priv->hw->pcs == STMMAC_PCS_RTBI)
	405	return false;
	406
	407	/* Check if MAC core supports the EEE feature. */
	408	if (!priv->dma_cap.eee)
	409	return false;
	410
	411	mutex_lock(&priv->lock);
	412
	413	/* Check if it needs to be deactivated */
	414	if (!priv->eee_active) {
	415	if (priv->eee_enabled) {
	416	netdev_dbg(priv->dev, "disable EEE\n");
	417	del_timer_sync(&priv->eee_ctrl_timer);
	418	stmmac_set_eee_timer(priv, priv->hw, 0, eee_tw_timer);
	419	}
	420	mutex_unlock(&priv->lock);
	421	return false;
	422	}
	423
	424	if (priv->eee_active && !priv->eee_enabled) {
	425	timer_setup(&priv->eee_ctrl_timer, stmmac_eee_ctrl_timer, 0);
	426	stmmac_set_eee_timer(priv, priv->hw, STMMAC_DEFAULT_LIT_LS,
	427	eee_tw_timer);
	428	}
	429
	430	mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer));
	431
	432	mutex_unlock(&priv->lock);
	433	netdev_dbg(priv->dev, "Energy-Efficient Ethernet initialized\n");
	434	return true;
	435	}
	436
	437	/* stmmac_get_tx_hwtstamp - get HW TX timestamps
	438	* @priv: driver private structure
	439	* @p : descriptor pointer
	440	* @skb : the socket buffer
	441	* Description :
	442	* This function will read timestamp from the descriptor & pass it to stack.
	443	* and also perform some sanity checks.
	444	*/
	445	static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv,
	446	struct dma_desc p, struct sk_buff skb)
	447	{
	448	struct skb_shared_hwtstamps shhwtstamp;
	449	bool found = false;
	450	u64 ns = 0;
	451
	452	if (!priv->hwts_tx_en)
	453	return;
	454
	455	/* exit if skb doesn't support hw tstamp */
	456	if (likely(!skb \|\| !(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS)))
	457	return;
	458
	459	/* check tx tstamp status */
	460	if (stmmac_get_tx_timestamp_status(priv, p)) {
	461	stmmac_get_timestamp(priv, p, priv->adv_ts, &ns);
	462	found = true;
	463	} else if (!stmmac_get_mac_tx_timestamp(priv, priv->hw, &ns)) {
	464	found = true;
	465	}
	466
	467	if (found) {
	468	memset(&shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps));
	469	shhwtstamp.hwtstamp = ns_to_ktime(ns);
	470
	471	netdev_dbg(priv->dev, "get valid TX hw timestamp %llu\n", ns);
	472	/* pass tstamp to stack */
	473	skb_tstamp_tx(skb, &shhwtstamp);
	474	}
	475	}
	476
	477	/* stmmac_get_rx_hwtstamp - get HW RX timestamps
	478	* @priv: driver private structure
	479	* @p : descriptor pointer
	480	* @np : next descriptor pointer
	481	* @skb : the socket buffer
	482	* Description :
	483	* This function will read received packet's timestamp from the descriptor
	484	* and pass it to stack. It also perform some sanity checks.
	485	*/
	486	static void stmmac_get_rx_hwtstamp(struct stmmac_priv priv, struct dma_desc p,
	487	struct dma_desc np, struct sk_buff skb)
	488	{
	489	struct skb_shared_hwtstamps *shhwtstamp = NULL;
	490	struct dma_desc *desc = p;
	491	u64 ns = 0;
	492
	493	if (!priv->hwts_rx_en)
	494	return;
	495	/* For GMAC4, the valid timestamp is from CTX next desc. */
	496	if (priv->plat->has_gmac4 \|\| priv->plat->has_xgmac)
	497	desc = np;
	498
	499	/* Check if timestamp is available */
	500	if (stmmac_get_rx_timestamp_status(priv, p, np, priv->adv_ts)) {
	501	stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns);
	502	netdev_dbg(priv->dev, "get valid RX hw timestamp %llu\n", ns);
	503	shhwtstamp = skb_hwtstamps(skb);
	504	memset(shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps));
	505	shhwtstamp->hwtstamp = ns_to_ktime(ns);
	506	} else {
	507	netdev_dbg(priv->dev, "cannot get RX hw timestamp\n");
	508	}
	509	}
	510
	511	/**
	512	* stmmac_hwtstamp_set - control hardware timestamping.
	513	* @dev: device pointer.
	514	* @ifr: An IOCTL specific structure, that can contain a pointer to
	515	* a proprietary structure used to pass information to the driver.
	516	* Description:
	517	* This function configures the MAC to enable/disable both outgoing(TX)
	518	* and incoming(RX) packets time stamping based on user input.
	519	* Return Value:
	520	* 0 on success and an appropriate -ve integer on failure.
	521	*/
	522	static int stmmac_hwtstamp_set(struct net_device dev, struct ifreq ifr)
	523	{
	524	struct stmmac_priv *priv = netdev_priv(dev);
	525	struct hwtstamp_config config;
	526	u32 ptp_v2 = 0;
	527	u32 tstamp_all = 0;
	528	u32 ptp_over_ipv4_udp = 0;
	529	u32 ptp_over_ipv6_udp = 0;
	530	u32 ptp_over_ethernet = 0;
	531	u32 snap_type_sel = 0;
	532	u32 ts_master_en = 0;
	533	u32 ts_event_en = 0;
	534
	535	if (!(priv->dma_cap.time_stamp \|\| priv->adv_ts)) {
	536	netdev_alert(priv->dev, "No support for HW time stamping\n");
	537	priv->hwts_tx_en = 0;
	538	priv->hwts_rx_en = 0;
	539
	540	return -EOPNOTSUPP;
	541	}
	542
	543	if (copy_from_user(&config, ifr->ifr_data,
	544	sizeof(config)))
	545	return -EFAULT;
	546
	547	netdev_dbg(priv->dev, "%s config flags:0x%x, tx_type:0x%x, rx_filter:0x%x\n",
	548	__func__, config.flags, config.tx_type, config.rx_filter);
	549
	550	/* reserved for future extensions */
	551	if (config.flags)
	552	return -EINVAL;
	553
	554	if (config.tx_type != HWTSTAMP_TX_OFF &&
	555	config.tx_type != HWTSTAMP_TX_ON)
	556	return -ERANGE;
	557
	558	if (priv->adv_ts) {
	559	switch (config.rx_filter) {
	560	case HWTSTAMP_FILTER_NONE:
	561	/* time stamp no incoming packet at all */
	562	config.rx_filter = HWTSTAMP_FILTER_NONE;
	563	break;
	564
	565	case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
	566	/* PTP v1, UDP, any kind of event packet */
	567	config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
	568	/* 'xmac' hardware can support Sync, Pdelay_Req and
	569	* Pdelay_resp by setting bit14 and bits17/16 to 01
	570	* This leaves Delay_Req timestamps out.
	571	* Enable all events and general purpose message
	572	* timestamping
	573	*/
	574	snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
	575	ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
	576	ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
	577	break;
	578
	579	case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
	580	/* PTP v1, UDP, Sync packet */
	581	config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_SYNC;
	582	/* take time stamp for SYNC messages only */
	583	ts_event_en = PTP_TCR_TSEVNTENA;
	584
	585	ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
	586	ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
	587	break;
	588
	589	case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
	590	/* PTP v1, UDP, Delay_req packet */
	591	config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ;
	592	/* take time stamp for Delay_Req messages only */
	593	ts_master_en = PTP_TCR_TSMSTRENA;
	594	ts_event_en = PTP_TCR_TSEVNTENA;
	595
	596	ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
	597	ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
	598	break;
	599
	600	case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
	601	/* PTP v2, UDP, any kind of event packet */
	602	config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT;
	603	ptp_v2 = PTP_TCR_TSVER2ENA;
	604	/* take time stamp for all event messages */
	605	snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
	606
	607	ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
	608	ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
	609	break;
	610
	611	case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
	612	/* PTP v2, UDP, Sync packet */
	613	config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_SYNC;
	614	ptp_v2 = PTP_TCR_TSVER2ENA;
	615	/* take time stamp for SYNC messages only */
	616	ts_event_en = PTP_TCR_TSEVNTENA;
	617
	618	ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
	619	ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
	620	break;
	621
	622	case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
	623	/* PTP v2, UDP, Delay_req packet */
	624	config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ;
	625	ptp_v2 = PTP_TCR_TSVER2ENA;
	626	/* take time stamp for Delay_Req messages only */
	627	ts_master_en = PTP_TCR_TSMSTRENA;
	628	ts_event_en = PTP_TCR_TSEVNTENA;
	629
	630	ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
	631	ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
	632	break;
	633
	634	case HWTSTAMP_FILTER_PTP_V2_EVENT:
	635	/* PTP v2/802.AS1 any layer, any kind of event packet */
	636	config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
	637	ptp_v2 = PTP_TCR_TSVER2ENA;
	638	snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
	639	if (priv->synopsys_id < DWMAC_CORE_4_10)
	640	ts_event_en = PTP_TCR_TSEVNTENA;
	641	ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
	642	ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
	643	ptp_over_ethernet = PTP_TCR_TSIPENA;
	644	break;
	645
	646	case HWTSTAMP_FILTER_PTP_V2_SYNC:
	647	/* PTP v2/802.AS1, any layer, Sync packet */
	648	config.rx_filter = HWTSTAMP_FILTER_PTP_V2_SYNC;
	649	ptp_v2 = PTP_TCR_TSVER2ENA;
	650	/* take time stamp for SYNC messages only */
	651	ts_event_en = PTP_TCR_TSEVNTENA;
	652
	653	ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
	654	ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
	655	ptp_over_ethernet = PTP_TCR_TSIPENA;
	656	break;
	657
	658	case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
	659	/* PTP v2/802.AS1, any layer, Delay_req packet */
	660	config.rx_filter = HWTSTAMP_FILTER_PTP_V2_DELAY_REQ;
	661	ptp_v2 = PTP_TCR_TSVER2ENA;
	662	/* take time stamp for Delay_Req messages only */
	663	ts_master_en = PTP_TCR_TSMSTRENA;
	664	ts_event_en = PTP_TCR_TSEVNTENA;
	665
	666	ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
	667	ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
	668	ptp_over_ethernet = PTP_TCR_TSIPENA;
	669	break;
	670
	671	case HWTSTAMP_FILTER_NTP_ALL:
	672	case HWTSTAMP_FILTER_ALL:
	673	/* time stamp any incoming packet */
	674	config.rx_filter = HWTSTAMP_FILTER_ALL;
	675	tstamp_all = PTP_TCR_TSENALL;
	676	break;
	677
	678	default:
	679	return -ERANGE;
	680	}
	681	} else {
	682	switch (config.rx_filter) {
	683	case HWTSTAMP_FILTER_NONE:
	684	config.rx_filter = HWTSTAMP_FILTER_NONE;
	685	break;
	686	default:
	687	/* PTP v1, UDP, any kind of event packet */
	688	config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
	689	break;
	690	}
	691	}
	692	priv->hwts_rx_en = ((config.rx_filter == HWTSTAMP_FILTER_NONE) ? 0 : 1);
	693	priv->hwts_tx_en = config.tx_type == HWTSTAMP_TX_ON;
	694
	695	priv->systime_flags = STMMAC_HWTS_ACTIVE;
	696
	697	if (priv->hwts_tx_en \|\| priv->hwts_rx_en) {
	698	priv->systime_flags \|= tstamp_all \| ptp_v2 \|
	699	ptp_over_ethernet \| ptp_over_ipv6_udp \|
	700	ptp_over_ipv4_udp \| ts_event_en \|
	701	ts_master_en \| snap_type_sel;
	702	}
	703
	704	stmmac_config_hw_tstamping(priv, priv->ptpaddr, priv->systime_flags);
	705
	706	memcpy(&priv->tstamp_config, &config, sizeof(config));
	707
	708	return copy_to_user(ifr->ifr_data, &config,
	709	sizeof(config)) ? -EFAULT : 0;
	710	}
	711
	712	/**
	713	* stmmac_hwtstamp_get - read hardware timestamping.
	714	* @dev: device pointer.
	715	* @ifr: An IOCTL specific structure, that can contain a pointer to
	716	* a proprietary structure used to pass information to the driver.
	717	* Description:
	718	* This function obtain the current hardware timestamping settings
	719	* as requested.
	720	*/
	721	static int stmmac_hwtstamp_get(struct net_device dev, struct ifreq ifr)
	722	{
	723	struct stmmac_priv *priv = netdev_priv(dev);
	724	struct hwtstamp_config *config = &priv->tstamp_config;
	725
	726	if (!(priv->dma_cap.time_stamp \|\| priv->dma_cap.atime_stamp))
	727	return -EOPNOTSUPP;
	728
	729	return copy_to_user(ifr->ifr_data, config,
	730	sizeof(*config)) ? -EFAULT : 0;
	731	}
	732
	733	/**
	734	* stmmac_init_tstamp_counter - init hardware timestamping counter
	735	* @priv: driver private structure
	736	* @systime_flags: timestamping flags
	737	* Description:
	738	* Initialize hardware counter for packet timestamping.
	739	* This is valid as long as the interface is open and not suspended.
	740	* Will be rerun after resuming from suspend, case in which the timestamping
	741	* flags updated by stmmac_hwtstamp_set() also need to be restored.
	742	*/
	743	int stmmac_init_tstamp_counter(struct stmmac_priv *priv, u32 systime_flags)
	744	{
	745	bool xmac = priv->plat->has_gmac4 \|\| priv->plat->has_xgmac;
	746	struct timespec64 now;
	747	u32 sec_inc = 0;
	748	u64 temp = 0;
	749
	750	if (!(priv->dma_cap.time_stamp \|\| priv->dma_cap.atime_stamp))
	751	return -EOPNOTSUPP;
	752
	753	stmmac_config_hw_tstamping(priv, priv->ptpaddr, systime_flags);
	754	priv->systime_flags = systime_flags;
	755
	756	/* program Sub Second Increment reg */
	757	stmmac_config_sub_second_increment(priv, priv->ptpaddr,
	758	priv->plat->clk_ptp_rate,
	759	xmac, &sec_inc);
	760	temp = div_u64(1000000000ULL, sec_inc);
	761
	762	/* Store sub second increment for later use */
	763	priv->sub_second_inc = sec_inc;
	764
	765	/* calculate default added value:
	766	* formula is :
	767	* addend = (2^32)/freq_div_ratio;
	768	* where, freq_div_ratio = 1e9ns/sec_inc
	769	*/
	770	temp = (u64)(temp << 32);
	771	priv->default_addend = div_u64(temp, priv->plat->clk_ptp_rate);
	772	stmmac_config_addend(priv, priv->ptpaddr, priv->default_addend);
	773
	774	/* initialize system time */
	775	ktime_get_real_ts64(&now);
	776
	777	/* lower 32 bits of tv_sec are safe until y2106 */
	778	stmmac_init_systime(priv, priv->ptpaddr, (u32)now.tv_sec, now.tv_nsec);
	779
	780	return 0;
	781	}
	782	EXPORT_SYMBOL_GPL(stmmac_init_tstamp_counter);
	783
	784	/**
	785	* stmmac_init_ptp - init PTP
	786	* @priv: driver private structure
	787	* Description: this is to verify if the HW supports the PTPv1 or PTPv2.
	788	* This is done by looking at the HW cap. register.
	789	* This function also registers the ptp driver.
	790	*/
	791	static int stmmac_init_ptp(struct stmmac_priv *priv)
	792	{
	793	bool xmac = priv->plat->has_gmac4 \|\| priv->plat->has_xgmac;
	794	int ret;
	795
	796	ret = stmmac_init_tstamp_counter(priv, STMMAC_HWTS_ACTIVE);
	797	if (ret)
	798	return ret;
	799
	800	priv->adv_ts = 0;
	801	/* Check if adv_ts can be enabled for dwmac 4.x / xgmac core */
	802	if (xmac && priv->dma_cap.atime_stamp)
	803	priv->adv_ts = 1;
	804	/* Dwmac 3.x core with extend_desc can support adv_ts */
	805	else if (priv->extend_desc && priv->dma_cap.atime_stamp)
	806	priv->adv_ts = 1;
	807
	808	if (priv->dma_cap.time_stamp)
	809	netdev_info(priv->dev, "IEEE 1588-2002 Timestamp supported\n");
	810
	811	if (priv->adv_ts)
	812	netdev_info(priv->dev,
	813	"IEEE 1588-2008 Advanced Timestamp supported\n");
	814
	815	priv->hwts_tx_en = 0;
	816	priv->hwts_rx_en = 0;
	817
	818	return 0;
	819	}
	820
	821	static void stmmac_release_ptp(struct stmmac_priv *priv)
	822	{
	823	clk_disable_unprepare(priv->plat->clk_ptp_ref);
	824	stmmac_ptp_unregister(priv);
	825	}
	826
	827	/**
	828	* stmmac_mac_flow_ctrl - Configure flow control in all queues
	829	* @priv: driver private structure
	830	* @duplex: duplex passed to the next function
	831	* Description: It is used for configuring the flow control in all queues
	832	*/
	833	static void stmmac_mac_flow_ctrl(struct stmmac_priv *priv, u32 duplex)
	834	{
	835	u32 tx_cnt = priv->plat->tx_queues_to_use;
	836
	837	stmmac_flow_ctrl(priv, priv->hw, duplex, priv->flow_ctrl & priv->plat->flow_ctrl,
	838	priv->pause, tx_cnt);
	839	}
	840
	841	static void stmmac_validate(struct phylink_config *config,
	842	unsigned long *supported,
	843	struct phylink_link_state *state)
	844	{
	845	struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));
	846	__ETHTOOL_DECLARE_LINK_MODE_MASK(mac_supported) = { 0, };
	847	__ETHTOOL_DECLARE_LINK_MODE_MASK(mask) = { 0, };
	848	int tx_cnt = priv->plat->tx_queues_to_use;
	849	int max_speed = priv->plat->max_speed;
	850
	851	phylink_set(mac_supported, 10baseT_Half);
	852	phylink_set(mac_supported, 10baseT_Full);
	853	phylink_set(mac_supported, 100baseT_Half);
	854	phylink_set(mac_supported, 100baseT_Full);
	855	phylink_set(mac_supported, 1000baseT_Half);
	856	phylink_set(mac_supported, 1000baseT_Full);
	857	phylink_set(mac_supported, 1000baseKX_Full);
	858	phylink_set(mac_supported, 100baseT1_Full);
	859	phylink_set(mac_supported, 1000baseT1_Full);
	860
	861	phylink_set(mac_supported, Autoneg);
	862	phylink_set(mac_supported, Pause);
	863	phylink_set(mac_supported, Asym_Pause);
	864	phylink_set_port_modes(mac_supported);
	865
	866	/* Cut down 1G if asked to */
	867	if ((max_speed > 0) && (max_speed < 1000)) {
	868	phylink_set(mask, 1000baseT_Full);
	869	phylink_set(mask, 1000baseX_Full);
	870	} else if (priv->plat->has_xgmac) {
	871	if (!max_speed \|\| (max_speed >= 2500)) {
	872	phylink_set(mac_supported, 2500baseT_Full);
	873	phylink_set(mac_supported, 2500baseX_Full);
	874	}
	875	if (!max_speed \|\| (max_speed >= 5000)) {
	876	phylink_set(mac_supported, 5000baseT_Full);
	877	}
	878	if (!max_speed \|\| (max_speed >= 10000)) {
	879	phylink_set(mac_supported, 10000baseSR_Full);
	880	phylink_set(mac_supported, 10000baseLR_Full);
	881	phylink_set(mac_supported, 10000baseER_Full);
	882	phylink_set(mac_supported, 10000baseLRM_Full);
	883	phylink_set(mac_supported, 10000baseT_Full);
	884	phylink_set(mac_supported, 10000baseKX4_Full);
	885	phylink_set(mac_supported, 10000baseKR_Full);
	886	}
	887	if (!max_speed \|\| (max_speed >= 25000)) {
	888	phylink_set(mac_supported, 25000baseCR_Full);
	889	phylink_set(mac_supported, 25000baseKR_Full);
	890	phylink_set(mac_supported, 25000baseSR_Full);
	891	}
	892	if (!max_speed \|\| (max_speed >= 40000)) {
	893	phylink_set(mac_supported, 40000baseKR4_Full);
	894	phylink_set(mac_supported, 40000baseCR4_Full);
	895	phylink_set(mac_supported, 40000baseSR4_Full);
	896	phylink_set(mac_supported, 40000baseLR4_Full);
	897	}
	898	if (!max_speed \|\| (max_speed >= 50000)) {
	899	phylink_set(mac_supported, 50000baseCR2_Full);
	900	phylink_set(mac_supported, 50000baseKR2_Full);
	901	phylink_set(mac_supported, 50000baseSR2_Full);
	902	phylink_set(mac_supported, 50000baseKR_Full);
	903	phylink_set(mac_supported, 50000baseSR_Full);
	904	phylink_set(mac_supported, 50000baseCR_Full);
	905	phylink_set(mac_supported, 50000baseLR_ER_FR_Full);
	906	phylink_set(mac_supported, 50000baseDR_Full);
	907	}
	908	if (!max_speed \|\| (max_speed >= 100000)) {
	909	phylink_set(mac_supported, 100000baseKR4_Full);
	910	phylink_set(mac_supported, 100000baseSR4_Full);
	911	phylink_set(mac_supported, 100000baseCR4_Full);
	912	phylink_set(mac_supported, 100000baseLR4_ER4_Full);
	913	phylink_set(mac_supported, 100000baseKR2_Full);
	914	phylink_set(mac_supported, 100000baseSR2_Full);
	915	phylink_set(mac_supported, 100000baseCR2_Full);
	916	phylink_set(mac_supported, 100000baseLR2_ER2_FR2_Full);
	917	phylink_set(mac_supported, 100000baseDR2_Full);
	918	}
	919	}
	920
	921	/* Half-Duplex can only work with single queue */
	922	if (tx_cnt > 1) {
	923	phylink_set(mask, 10baseT_Half);
	924	phylink_set(mask, 100baseT_Half);
	925	phylink_set(mask, 1000baseT_Half);
	926	}
	927
	928	linkmode_and(supported, supported, mac_supported);
	929	linkmode_andnot(supported, supported, mask);
	930
	931	linkmode_and(state->advertising, state->advertising, mac_supported);
	932	linkmode_andnot(state->advertising, state->advertising, mask);
	933
	934	/* If PCS is supported, check which modes it supports. */
	935	stmmac_xpcs_validate(priv, &priv->hw->xpcs_args, supported, state);
	936	}
	937
	938	static void stmmac_mac_pcs_get_state(struct phylink_config *config,
	939	struct phylink_link_state *state)
	940	{
	941	struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));
	942
	943	state->link = 0;
	944	stmmac_xpcs_get_state(priv, &priv->hw->xpcs_args, state);
	945	}
	946
	947	static void stmmac_mac_config(struct phylink_config *config, unsigned int mode,
	948	const struct phylink_link_state *state)
	949	{
	950	struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));
	951
	952	stmmac_xpcs_config(priv, &priv->hw->xpcs_args, state);
	953	}
	954
	955	static void stmmac_mac_an_restart(struct phylink_config *config)
	956	{
	957	/* Not Supported */
	958	}
	959
	960	static void stmmac_mac_link_down(struct phylink_config *config,
	961	unsigned int mode, phy_interface_t interface)
	962	{
	963	struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));
	964
	965	stmmac_mac_set(priv, priv->ioaddr, false);
	966	priv->eee_active = false;
	967	priv->tx_lpi_enabled = false;
	968	stmmac_eee_init(priv);
	969	stmmac_set_eee_pls(priv, priv->hw, false);
	970	}
	971
	972	static void stmmac_mac_link_up(struct phylink_config *config,
	973	struct phy_device *phy,
	974	unsigned int mode, phy_interface_t interface,
	975	int speed, int duplex,
	976	bool tx_pause, bool rx_pause)
	977	{
	978	struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));
	979	u32 ctrl;
	980
	981	stmmac_xpcs_link_up(priv, &priv->hw->xpcs_args, speed, interface);
	982
	983	ctrl = readl(priv->ioaddr + MAC_CTRL_REG);
	984	ctrl &= ~priv->hw->link.speed_mask;
	985
	986	if (interface == PHY_INTERFACE_MODE_USXGMII) {
	987	switch (speed) {
	988	case SPEED_10000:
	989	ctrl \|= priv->hw->link.xgmii.speed10000;
	990	break;
	991	case SPEED_5000:
	992	ctrl \|= priv->hw->link.xgmii.speed5000;
	993	break;
	994	case SPEED_2500:
	995	ctrl \|= priv->hw->link.xgmii.speed2500;
	996	break;
	997	default:
	998	return;
	999	}
	1000	} else if (interface == PHY_INTERFACE_MODE_XLGMII) {
	1001	switch (speed) {
	1002	case SPEED_100000:
	1003	ctrl \|= priv->hw->link.xlgmii.speed100000;
	1004	break;
	1005	case SPEED_50000:
	1006	ctrl \|= priv->hw->link.xlgmii.speed50000;
	1007	break;
	1008	case SPEED_40000:
	1009	ctrl \|= priv->hw->link.xlgmii.speed40000;
	1010	break;
	1011	case SPEED_25000:
	1012	ctrl \|= priv->hw->link.xlgmii.speed25000;
	1013	break;
	1014	case SPEED_10000:
	1015	ctrl \|= priv->hw->link.xgmii.speed10000;
	1016	break;
	1017	case SPEED_2500:
	1018	ctrl \|= priv->hw->link.speed2500;
	1019	break;
	1020	case SPEED_1000:
	1021	ctrl \|= priv->hw->link.speed1000;
	1022	break;
	1023	default:
	1024	return;
	1025	}
	1026	} else {
	1027	switch (speed) {
	1028	case SPEED_2500:
	1029	ctrl \|= priv->hw->link.speed2500;
	1030	break;
	1031	case SPEED_1000:
	1032	ctrl \|= priv->hw->link.speed1000;
	1033	break;
	1034	case SPEED_100:
	1035	ctrl \|= priv->hw->link.speed100;
	1036	break;
	1037	case SPEED_10:
	1038	ctrl \|= priv->hw->link.speed10;
	1039	break;
	1040	default:
	1041	return;
	1042	}
	1043	}
	1044
	1045	priv->speed = speed;
	1046
	1047	if (priv->plat->fix_mac_speed)
	1048	priv->plat->fix_mac_speed(priv->plat->bsp_priv, speed);
	1049
	1050	if (!duplex)
	1051	ctrl &= ~priv->hw->link.duplex;
	1052	else
	1053	ctrl \|= priv->hw->link.duplex;
	1054
	1055	/* Flow Control operation */
	1056	if (rx_pause && tx_pause)
	1057	priv->flow_ctrl = FLOW_AUTO;
	1058	else if (rx_pause && !tx_pause)
	1059	priv->flow_ctrl = FLOW_RX;
	1060	else if (!rx_pause && tx_pause)
	1061	priv->flow_ctrl = FLOW_TX;
	1062	else
	1063	priv->flow_ctrl = FLOW_OFF;
	1064
	1065	stmmac_mac_flow_ctrl(priv, duplex);
	1066
	1067	writel(ctrl, priv->ioaddr + MAC_CTRL_REG);
	1068
	1069	stmmac_mac_set(priv, priv->ioaddr, true);
	1070	if (phy && priv->dma_cap.eee) {
	1071	priv->eee_active = phy_init_eee(phy, 1) >= 0;
	1072	priv->eee_enabled = stmmac_eee_init(priv);
	1073	priv->tx_lpi_enabled = priv->eee_enabled;
	1074	stmmac_set_eee_pls(priv, priv->hw, true);
	1075	}
	1076	}
	1077
	1078	static const struct phylink_mac_ops stmmac_phylink_mac_ops = {
	1079	.validate = stmmac_validate,
	1080	.mac_pcs_get_state = stmmac_mac_pcs_get_state,
	1081	.mac_config = stmmac_mac_config,
	1082	.mac_an_restart = stmmac_mac_an_restart,
	1083	.mac_link_down = stmmac_mac_link_down,
	1084	.mac_link_up = stmmac_mac_link_up,
	1085	};
	1086
	1087	/**
	1088	* stmmac_check_pcs_mode - verify if RGMII/SGMII is supported
	1089	* @priv: driver private structure
	1090	* Description: this is to verify if the HW supports the PCS.
	1091	* Physical Coding Sublayer (PCS) interface that can be used when the MAC is
	1092	* configured for the TBI, RTBI, or SGMII PHY interface.
	1093	*/
	1094	static void stmmac_check_pcs_mode(struct stmmac_priv *priv)
	1095	{
	1096	int interface = priv->plat->interface;
	1097
	1098	if (priv->dma_cap.pcs) {
	1099	if ((interface == PHY_INTERFACE_MODE_RGMII) \|\|
	1100	(interface == PHY_INTERFACE_MODE_RGMII_ID) \|\|
	1101	(interface == PHY_INTERFACE_MODE_RGMII_RXID) \|\|
	1102	(interface == PHY_INTERFACE_MODE_RGMII_TXID)) {
	1103	netdev_dbg(priv->dev, "PCS RGMII support enabled\n");
	1104	priv->hw->pcs = STMMAC_PCS_RGMII;
	1105	} else if (interface == PHY_INTERFACE_MODE_SGMII) {
	1106	netdev_dbg(priv->dev, "PCS SGMII support enabled\n");
	1107	priv->hw->pcs = STMMAC_PCS_SGMII;
	1108	}
	1109	}
	1110	}
	1111
	1112	/**
	1113	* stmmac_init_phy - PHY initialization
	1114	* @dev: net device structure
	1115	* Description: it initializes the driver's PHY state, and attaches the PHY
	1116	* to the mac driver.
	1117	* Return value:
	1118	* 0 on success
	1119	*/
	1120	static int stmmac_init_phy(struct net_device *dev)
	1121	{
	1122	struct stmmac_priv *priv = netdev_priv(dev);
	1123	struct device_node *node;
	1124	int ret;
	1125
	1126	if (priv->plat->integrated_phy_power)
	1127	ret = priv->plat->integrated_phy_power(priv->plat->bsp_priv, true);
	1128
	1129	node = priv->plat->phylink_node;
	1130
	1131	if (node)
	1132	ret = phylink_of_phy_connect(priv->phylink, node, 0);
	1133
	1134	/* Some DT bindings do not set-up the PHY handle. Let's try to
	1135	* manually parse it
	1136	*/
	1137	if (!node \|\| ret) {
	1138	int addr = priv->plat->phy_addr;
	1139	struct phy_device *phydev;
	1140
	1141	phydev = mdiobus_get_phy(priv->mii, addr);
	1142	if (!phydev) {
	1143	netdev_err(priv->dev, "no phy at addr %d\n", addr);
	1144	return -ENODEV;
	1145	}
	1146
	1147	ret = phylink_connect_phy(priv->phylink, phydev);
	1148	}
	1149
	1150	if (!priv->plat->pmt) {
	1151	struct ethtool_wolinfo wol = { .cmd = ETHTOOL_GWOL };
	1152
	1153	phylink_ethtool_get_wol(priv->phylink, &wol);
	1154	device_set_wakeup_capable(priv->device, !!wol.supported);
	1155	}
	1156
	1157	return ret;
	1158	}
	1159
	1160	static int stmmac_phy_setup(struct stmmac_priv *priv)
	1161	{
	1162	struct fwnode_handle *fwnode = of_fwnode_handle(priv->plat->phylink_node);
	1163	int mode = priv->plat->phy_interface;
	1164	struct phylink *phylink;
	1165
	1166	priv->phylink_config.dev = &priv->dev->dev;
	1167	priv->phylink_config.type = PHYLINK_NETDEV;
	1168	priv->phylink_config.pcs_poll = true;
	1169
	1170	if (!fwnode)
	1171	fwnode = dev_fwnode(priv->device);
	1172
	1173	phylink = phylink_create(&priv->phylink_config, fwnode,
	1174	mode, &stmmac_phylink_mac_ops);
	1175	if (IS_ERR(phylink))
	1176	return PTR_ERR(phylink);
	1177
	1178	priv->phylink = phylink;
	1179	return 0;
	1180	}
	1181
	1182	static void stmmac_display_rx_rings(struct stmmac_priv *priv)
	1183	{
	1184	u32 rx_cnt = priv->plat->rx_queues_to_use;
	1185	unsigned int desc_size;
	1186	void *head_rx;
	1187	u32 queue;
	1188
	1189	/* Display RX rings */
	1190	for (queue = 0; queue < rx_cnt; queue++) {
	1191	struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
	1192
	1193	pr_info("\tRX Queue %u rings\n", queue);
	1194
	1195	if (priv->extend_desc) {
	1196	head_rx = (void *)rx_q->dma_erx;
	1197	desc_size = sizeof(struct dma_extended_desc);
	1198	} else {
	1199	head_rx = (void *)rx_q->dma_rx;
	1200	desc_size = sizeof(struct dma_desc);
	1201	}
	1202
	1203	/* Display RX ring */
	1204	stmmac_display_ring(priv, head_rx, priv->dma_rx_size, true,
	1205	rx_q->dma_rx_phy, desc_size);
	1206	}
	1207	}
	1208
	1209	static void stmmac_display_tx_rings(struct stmmac_priv *priv)
	1210	{
	1211	u32 tx_cnt = priv->plat->tx_queues_to_use;
	1212	unsigned int desc_size;
	1213	void *head_tx;
	1214	u32 queue;
	1215
	1216	/* Display TX rings */
	1217	for (queue = 0; queue < tx_cnt; queue++) {
	1218	struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
	1219
	1220	pr_info("\tTX Queue %d rings\n", queue);
	1221
	1222	if (priv->extend_desc) {
	1223	head_tx = (void *)tx_q->dma_etx;
	1224	desc_size = sizeof(struct dma_extended_desc);
	1225	} else if (tx_q->tbs & STMMAC_TBS_AVAIL) {
	1226	head_tx = (void *)tx_q->dma_entx;
	1227	desc_size = sizeof(struct dma_edesc);
	1228	} else {
	1229	head_tx = (void *)tx_q->dma_tx;
	1230	desc_size = sizeof(struct dma_desc);
	1231	}
	1232
	1233	stmmac_display_ring(priv, head_tx, priv->dma_tx_size, false,
	1234	tx_q->dma_tx_phy, desc_size);
	1235	}
	1236	}
	1237
	1238	static void stmmac_display_rings(struct stmmac_priv *priv)
	1239	{
	1240	/* Display RX ring */
	1241	stmmac_display_rx_rings(priv);
	1242
	1243	/* Display TX ring */
	1244	stmmac_display_tx_rings(priv);
	1245	}
	1246
	1247	static int stmmac_set_bfsize(int mtu, int bufsize)
	1248	{
	1249	int ret = bufsize;
	1250
	1251	if (mtu >= BUF_SIZE_8KiB)
	1252	ret = BUF_SIZE_16KiB;
	1253	else if (mtu >= BUF_SIZE_4KiB)
	1254	ret = BUF_SIZE_8KiB;
	1255	else if (mtu >= BUF_SIZE_2KiB)
	1256	ret = BUF_SIZE_4KiB;
	1257	else if (mtu > DEFAULT_BUFSIZE)
	1258	ret = BUF_SIZE_2KiB;
	1259	else
	1260	ret = DEFAULT_BUFSIZE;
	1261
	1262	return ret;
	1263	}
	1264
	1265	/**
	1266	* stmmac_clear_rx_descriptors - clear RX descriptors
	1267	* @priv: driver private structure
	1268	* @queue: RX queue index
	1269	* Description: this function is called to clear the RX descriptors
	1270	* in case of both basic and extended descriptors are used.
	1271	*/
	1272	static void stmmac_clear_rx_descriptors(struct stmmac_priv *priv, u32 queue)
	1273	{
	1274	struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
	1275	int i;
	1276
	1277	/* Clear the RX descriptors */
	1278	for (i = 0; i < priv->dma_rx_size; i++)
	1279	if (priv->extend_desc)
	1280	stmmac_init_rx_desc(priv, &rx_q->dma_erx[i].basic,
	1281	priv->use_riwt, priv->mode,
	1282	(i == priv->dma_rx_size - 1),
	1283	priv->dma_buf_sz);
	1284	else
	1285	stmmac_init_rx_desc(priv, &rx_q->dma_rx[i],
	1286	priv->use_riwt, priv->mode,
	1287	(i == priv->dma_rx_size - 1),
	1288	priv->dma_buf_sz);
	1289	}
	1290
	1291	/**
	1292	* stmmac_clear_tx_descriptors - clear tx descriptors
	1293	* @priv: driver private structure
	1294	* @queue: TX queue index.
	1295	* Description: this function is called to clear the TX descriptors
	1296	* in case of both basic and extended descriptors are used.
	1297	*/
	1298	static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv, u32 queue)
	1299	{
	1300	struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
	1301	int i;
	1302
	1303	/* Clear the TX descriptors */
	1304	for (i = 0; i < priv->dma_tx_size; i++) {
	1305	int last = (i == (priv->dma_tx_size - 1));
	1306	struct dma_desc *p;
	1307
	1308	if (priv->extend_desc)
	1309	p = &tx_q->dma_etx[i].basic;
	1310	else if (tx_q->tbs & STMMAC_TBS_AVAIL)
	1311	p = &tx_q->dma_entx[i].basic;
	1312	else
	1313	p = &tx_q->dma_tx[i];
	1314
	1315	stmmac_init_tx_desc(priv, p, priv->mode, last);
	1316	}
	1317	}
	1318
	1319	/**
	1320	* stmmac_clear_descriptors - clear descriptors
	1321	* @priv: driver private structure
	1322	* Description: this function is called to clear the TX and RX descriptors
	1323	* in case of both basic and extended descriptors are used.
	1324	*/
	1325	static void stmmac_clear_descriptors(struct stmmac_priv *priv)
	1326	{
	1327	u32 rx_queue_cnt = priv->plat->rx_queues_to_use;
	1328	u32 tx_queue_cnt = priv->plat->tx_queues_to_use;
	1329	u32 queue;
	1330
	1331	/* Clear the RX descriptors */
	1332	for (queue = 0; queue < rx_queue_cnt; queue++)
	1333	stmmac_clear_rx_descriptors(priv, queue);
	1334
	1335	/* Clear the TX descriptors */
	1336	for (queue = 0; queue < tx_queue_cnt; queue++)
	1337	stmmac_clear_tx_descriptors(priv, queue);
	1338	}
	1339
	1340	/**
	1341	* stmmac_init_rx_buffers - init the RX descriptor buffer.
	1342	* @priv: driver private structure
	1343	* @p: descriptor pointer
	1344	* @i: descriptor index
	1345	* @flags: gfp flag
	1346	* @queue: RX queue index
	1347	* Description: this function is called to allocate a receive buffer, perform
	1348	* the DMA mapping and init the descriptor.
	1349	*/
	1350	static int stmmac_init_rx_buffers(struct stmmac_priv priv, struct dma_desc p,
	1351	int i, gfp_t flags, u32 queue)
	1352	{
	1353	struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
	1354	struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i];
	1355	gfp_t gfp = (GFP_ATOMIC \| __GFP_NOWARN);
	1356
	1357	if (priv->dma_cap.addr64 <= 32)
	1358	gfp \|= GFP_DMA32;
	1359
	1360	buf->page = page_pool_alloc_pages(rx_q->page_pool, gfp);
	1361	if (!buf->page)
	1362	return -ENOMEM;
	1363
	1364	if (priv->sph) {
	1365	buf->sec_page = page_pool_alloc_pages(rx_q->page_pool, gfp);
	1366	if (!buf->sec_page)
	1367	return -ENOMEM;
	1368
	1369	buf->sec_addr = page_pool_get_dma_addr(buf->sec_page);
	1370	stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true);
	1371	} else {
	1372	buf->sec_page = NULL;
	1373	stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, false);
	1374	}
	1375
	1376	buf->addr = page_pool_get_dma_addr(buf->page);
	1377	stmmac_set_desc_addr(priv, p, buf->addr);
	1378	if (priv->dma_buf_sz == BUF_SIZE_16KiB)
	1379	stmmac_init_desc3(priv, p);
	1380
	1381	return 0;
	1382	}
	1383
	1384	/**
	1385	* stmmac_free_rx_buffer - free RX dma buffers
	1386	* @priv: private structure
	1387	* @queue: RX queue index
	1388	* @i: buffer index.
	1389	*/
	1390	static void stmmac_free_rx_buffer(struct stmmac_priv *priv, u32 queue, int i)
	1391	{
	1392	struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
	1393	struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i];
	1394
	1395	if (buf->page)
	1396	page_pool_put_full_page(rx_q->page_pool, buf->page, false);
	1397	buf->page = NULL;
	1398
	1399	if (buf->sec_page)
	1400	page_pool_put_full_page(rx_q->page_pool, buf->sec_page, false);
	1401	buf->sec_page = NULL;
	1402	}
	1403
	1404	/**
	1405	* stmmac_free_tx_buffer - free RX dma buffers
	1406	* @priv: private structure
	1407	* @queue: RX queue index
	1408	* @i: buffer index.
	1409	*/
	1410	static void stmmac_free_tx_buffer(struct stmmac_priv *priv, u32 queue, int i)
	1411	{
	1412	struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
	1413
	1414	if (tx_q->tx_skbuff_dma[i].buf) {
	1415	if (tx_q->tx_skbuff_dma[i].map_as_page)
	1416	dma_unmap_page(priv->device,
	1417	tx_q->tx_skbuff_dma[i].buf,
	1418	tx_q->tx_skbuff_dma[i].len,
	1419	DMA_TO_DEVICE);
	1420	else
	1421	dma_unmap_single(priv->device,
	1422	tx_q->tx_skbuff_dma[i].buf,
	1423	tx_q->tx_skbuff_dma[i].len,
	1424	DMA_TO_DEVICE);
	1425	}
	1426
	1427	if (tx_q->tx_skbuff[i]) {
	1428	dev_kfree_skb_any(tx_q->tx_skbuff[i]);
	1429	tx_q->tx_skbuff[i] = NULL;
	1430	tx_q->tx_skbuff_dma[i].buf = 0;
	1431	tx_q->tx_skbuff_dma[i].map_as_page = false;
	1432	}
	1433	}
	1434
	1435	/**
	1436	* init_dma_rx_desc_rings - init the RX descriptor rings
	1437	* @dev: net device structure
	1438	* @flags: gfp flag.
	1439	* Description: this function initializes the DMA RX descriptors
	1440	* and allocates the socket buffers. It supports the chained and ring
	1441	* modes.
	1442	*/
	1443	static int init_dma_rx_desc_rings(struct net_device *dev, gfp_t flags)
	1444	{
	1445	struct stmmac_priv *priv = netdev_priv(dev);
	1446	u32 rx_count = priv->plat->rx_queues_to_use;
	1447	int ret = -ENOMEM;
	1448	int queue;
	1449	int i;
	1450
	1451	/* RX INITIALIZATION */
	1452	netif_dbg(priv, probe, priv->dev,
	1453	"SKB addresses:\nskb\t\tskb data\tdma data\n");
	1454
	1455	for (queue = 0; queue < rx_count; queue++) {
	1456	struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
	1457
	1458	netif_dbg(priv, probe, priv->dev,
	1459	"(%s) dma_rx_phy=0x%08x\n", __func__,
	1460	(u32)rx_q->dma_rx_phy);
	1461
	1462	stmmac_clear_rx_descriptors(priv, queue);
	1463
	1464	for (i = 0; i < priv->dma_rx_size; i++) {
	1465	struct dma_desc *p;
	1466
	1467	if (priv->extend_desc)
	1468	p = &((rx_q->dma_erx + i)->basic);
	1469	else
	1470	p = rx_q->dma_rx + i;
	1471
	1472	ret = stmmac_init_rx_buffers(priv, p, i, flags,
	1473	queue);
	1474	if (ret)
	1475	goto err_init_rx_buffers;
	1476	}
	1477
	1478	rx_q->cur_rx = 0;
	1479	rx_q->dirty_rx = (unsigned int)(i - priv->dma_rx_size);
	1480
	1481	/* Setup the chained descriptor addresses */
	1482	if (priv->mode == STMMAC_CHAIN_MODE) {
	1483	if (priv->extend_desc)
	1484	stmmac_mode_init(priv, rx_q->dma_erx,
	1485	rx_q->dma_rx_phy,
	1486	priv->dma_rx_size, 1);
	1487	else
	1488	stmmac_mode_init(priv, rx_q->dma_rx,
	1489	rx_q->dma_rx_phy,
	1490	priv->dma_rx_size, 0);
	1491	}
	1492	}
	1493
	1494	return 0;
	1495
	1496	err_init_rx_buffers:
	1497	while (queue >= 0) {
	1498	while (--i >= 0)
	1499	stmmac_free_rx_buffer(priv, queue, i);
	1500
	1501	if (queue == 0)
	1502	break;
	1503
	1504	i = priv->dma_rx_size;
	1505	queue--;
	1506	}
	1507
	1508	return ret;
	1509	}
	1510
	1511	/**
	1512	* init_dma_tx_desc_rings - init the TX descriptor rings
	1513	* @dev: net device structure.
	1514	* Description: this function initializes the DMA TX descriptors
	1515	* and allocates the socket buffers. It supports the chained and ring
	1516	* modes.
	1517	*/
	1518	static int init_dma_tx_desc_rings(struct net_device *dev)
	1519	{
	1520	struct stmmac_priv *priv = netdev_priv(dev);
	1521	u32 tx_queue_cnt = priv->plat->tx_queues_to_use;
	1522	u32 queue;
	1523	int i;
	1524
	1525	for (queue = 0; queue < tx_queue_cnt; queue++) {
	1526	struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
	1527
	1528	netif_dbg(priv, probe, priv->dev,
	1529	"(%s) dma_tx_phy=0x%08x\n", __func__,
	1530	(u32)tx_q->dma_tx_phy);
	1531
	1532	/* Setup the chained descriptor addresses */
	1533	if (priv->mode == STMMAC_CHAIN_MODE) {
	1534	if (priv->extend_desc)
	1535	stmmac_mode_init(priv, tx_q->dma_etx,
	1536	tx_q->dma_tx_phy,
	1537	priv->dma_tx_size, 1);
	1538	else if (!(tx_q->tbs & STMMAC_TBS_AVAIL))
	1539	stmmac_mode_init(priv, tx_q->dma_tx,
	1540	tx_q->dma_tx_phy,
	1541	priv->dma_tx_size, 0);
	1542	}
	1543
	1544	for (i = 0; i < priv->dma_tx_size; i++) {
	1545	struct dma_desc *p;
	1546	if (priv->extend_desc)
	1547	p = &((tx_q->dma_etx + i)->basic);
	1548	else if (tx_q->tbs & STMMAC_TBS_AVAIL)
	1549	p = &((tx_q->dma_entx + i)->basic);
	1550	else
	1551	p = tx_q->dma_tx + i;
	1552
	1553	stmmac_clear_desc(priv, p);
	1554
	1555	tx_q->tx_skbuff_dma[i].buf = 0;
	1556	tx_q->tx_skbuff_dma[i].map_as_page = false;
	1557	tx_q->tx_skbuff_dma[i].len = 0;
	1558	tx_q->tx_skbuff_dma[i].last_segment = false;
	1559	tx_q->tx_skbuff[i] = NULL;
	1560	}
	1561
	1562	tx_q->dirty_tx = 0;
	1563	tx_q->cur_tx = 0;
	1564	tx_q->mss = 0;
	1565
	1566	netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, queue));
	1567	}
	1568
	1569	return 0;
	1570	}
	1571
	1572	/**
	1573	* init_dma_desc_rings - init the RX/TX descriptor rings
	1574	* @dev: net device structure
	1575	* @flags: gfp flag.
	1576	* Description: this function initializes the DMA RX/TX descriptors
	1577	* and allocates the socket buffers. It supports the chained and ring
	1578	* modes.
	1579	*/
	1580	static int init_dma_desc_rings(struct net_device *dev, gfp_t flags)
	1581	{
	1582	struct stmmac_priv *priv = netdev_priv(dev);
	1583	int ret;
	1584
	1585	ret = init_dma_rx_desc_rings(dev, flags);
	1586	if (ret)
	1587	return ret;
	1588
	1589	ret = init_dma_tx_desc_rings(dev);
	1590
	1591	stmmac_clear_descriptors(priv);
	1592
	1593	if (netif_msg_hw(priv))
	1594	stmmac_display_rings(priv);
	1595
	1596	return ret;
	1597	}
	1598
	1599	/**
	1600	* dma_free_rx_skbufs - free RX dma buffers
	1601	* @priv: private structure
	1602	* @queue: RX queue index
	1603	*/
	1604	static void dma_free_rx_skbufs(struct stmmac_priv *priv, u32 queue)
	1605	{
	1606	int i;
	1607
	1608	for (i = 0; i < priv->dma_rx_size; i++)
	1609	stmmac_free_rx_buffer(priv, queue, i);
	1610	}
	1611
	1612	/**
	1613	* dma_free_tx_skbufs - free TX dma buffers
	1614	* @priv: private structure
	1615	* @queue: TX queue index
	1616	*/
	1617	static void dma_free_tx_skbufs(struct stmmac_priv *priv, u32 queue)
	1618	{
	1619	int i;
	1620
	1621	for (i = 0; i < priv->dma_tx_size; i++)
	1622	stmmac_free_tx_buffer(priv, queue, i);
	1623	}
	1624
	1625	/**
	1626	* stmmac_free_tx_skbufs - free TX skb buffers
	1627	* @priv: private structure
	1628	*/
	1629	static void stmmac_free_tx_skbufs(struct stmmac_priv *priv)
	1630	{
	1631	u32 tx_queue_cnt = priv->plat->tx_queues_to_use;
	1632	u32 queue;
	1633
	1634	for (queue = 0; queue < tx_queue_cnt; queue++)
	1635	dma_free_tx_skbufs(priv, queue);
	1636	}
	1637
	1638	/**
	1639	* free_dma_rx_desc_resources - free RX dma desc resources
	1640	* @priv: private structure
	1641	*/
	1642	static void free_dma_rx_desc_resources(struct stmmac_priv *priv)
	1643	{
	1644	u32 rx_count = priv->plat->rx_queues_to_use;
	1645	u32 queue;
	1646
	1647	/* Free RX queue resources */
	1648	for (queue = 0; queue < rx_count; queue++) {
	1649	struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
	1650
	1651	/* Release the DMA RX socket buffers */
	1652	dma_free_rx_skbufs(priv, queue);
	1653
	1654	/* Free DMA regions of consistent memory previously allocated */
	1655	if (!priv->extend_desc)
	1656	dma_free_coherent(priv->device, priv->dma_rx_size *
	1657	sizeof(struct dma_desc),
	1658	rx_q->dma_rx, rx_q->dma_rx_phy);
	1659	else
	1660	dma_free_coherent(priv->device, priv->dma_rx_size *
	1661	sizeof(struct dma_extended_desc),
	1662	rx_q->dma_erx, rx_q->dma_rx_phy);
	1663
	1664	kfree(rx_q->buf_pool);
	1665	if (rx_q->page_pool)
	1666	page_pool_destroy(rx_q->page_pool);
	1667	}
	1668	}
	1669
	1670	/**
	1671	* free_dma_tx_desc_resources - free TX dma desc resources
	1672	* @priv: private structure
	1673	*/
	1674	static void free_dma_tx_desc_resources(struct stmmac_priv *priv)
	1675	{
	1676	u32 tx_count = priv->plat->tx_queues_to_use;
	1677	u32 queue;
	1678
	1679	/* Free TX queue resources */
	1680	for (queue = 0; queue < tx_count; queue++) {
	1681	struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
	1682	size_t size;
	1683	void *addr;
	1684
	1685	/* Release the DMA TX socket buffers */
	1686	dma_free_tx_skbufs(priv, queue);
	1687
	1688	if (priv->extend_desc) {
	1689	size = sizeof(struct dma_extended_desc);
	1690	addr = tx_q->dma_etx;
	1691	} else if (tx_q->tbs & STMMAC_TBS_AVAIL) {
	1692	size = sizeof(struct dma_edesc);
	1693	addr = tx_q->dma_entx;
	1694	} else {
	1695	size = sizeof(struct dma_desc);
	1696	addr = tx_q->dma_tx;
	1697	}
	1698
	1699	size *= priv->dma_tx_size;
	1700
	1701	dma_free_coherent(priv->device, size, addr, tx_q->dma_tx_phy);
	1702
	1703	kfree(tx_q->tx_skbuff_dma);
	1704	kfree(tx_q->tx_skbuff);
	1705	}
	1706	}
	1707
	1708	/**
	1709	* alloc_dma_rx_desc_resources - alloc RX resources.
	1710	* @priv: private structure
	1711	* Description: according to which descriptor can be used (extend or basic)
	1712	* this function allocates the resources for TX and RX paths. In case of
	1713	* reception, for example, it pre-allocated the RX socket buffer in order to
	1714	* allow zero-copy mechanism.
	1715	*/
	1716	static int alloc_dma_rx_desc_resources(struct stmmac_priv *priv)
	1717	{
	1718	u32 rx_count = priv->plat->rx_queues_to_use;
	1719	int ret = -ENOMEM;
	1720	u32 queue;
	1721
	1722	/* RX queues buffers and DMA */
	1723	for (queue = 0; queue < rx_count; queue++) {
	1724	struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
	1725	struct page_pool_params pp_params = { 0 };
	1726	unsigned int num_pages;
	1727
	1728	rx_q->queue_index = queue;
	1729	rx_q->priv_data = priv;
	1730
	1731	pp_params.flags = PP_FLAG_DMA_MAP \| PP_FLAG_DMA_SYNC_DEV;
	1732	pp_params.pool_size = priv->dma_rx_size;
	1733	num_pages = DIV_ROUND_UP(priv->dma_buf_sz, PAGE_SIZE);
	1734	pp_params.order = ilog2(num_pages);
	1735	pp_params.nid = dev_to_node(priv->device);
	1736	pp_params.dev = priv->device;
	1737	pp_params.dma_dir = DMA_FROM_DEVICE;
	1738	pp_params.max_len = num_pages * PAGE_SIZE;
	1739
	1740	rx_q->page_pool = page_pool_create(&pp_params);
	1741	if (IS_ERR(rx_q->page_pool)) {
	1742	ret = PTR_ERR(rx_q->page_pool);
	1743	rx_q->page_pool = NULL;
	1744	goto err_dma;
	1745	}
	1746
	1747	rx_q->buf_pool = kcalloc(priv->dma_rx_size,
	1748	sizeof(*rx_q->buf_pool),
	1749	GFP_KERNEL);
	1750	if (!rx_q->buf_pool)
	1751	goto err_dma;
	1752
	1753	if (priv->extend_desc) {
	1754	rx_q->dma_erx = dma_alloc_coherent(priv->device,
	1755	priv->dma_rx_size *
	1756	sizeof(struct dma_extended_desc),
	1757	&rx_q->dma_rx_phy,
	1758	GFP_KERNEL);
	1759	if (!rx_q->dma_erx)
	1760	goto err_dma;
	1761
	1762	} else {
	1763	rx_q->dma_rx = dma_alloc_coherent(priv->device,
	1764	priv->dma_rx_size *
	1765	sizeof(struct dma_desc),
	1766	&rx_q->dma_rx_phy,
	1767	GFP_KERNEL);
	1768	if (!rx_q->dma_rx)
	1769	goto err_dma;
	1770	}
	1771	}
	1772
	1773	return 0;
	1774
	1775	err_dma:
	1776	free_dma_rx_desc_resources(priv);
	1777
	1778	return ret;
	1779	}
	1780
	1781	/**
	1782	* alloc_dma_tx_desc_resources - alloc TX resources.
	1783	* @priv: private structure
	1784	* Description: according to which descriptor can be used (extend or basic)
	1785	* this function allocates the resources for TX and RX paths. In case of
	1786	* reception, for example, it pre-allocated the RX socket buffer in order to
	1787	* allow zero-copy mechanism.
	1788	*/
	1789	static int alloc_dma_tx_desc_resources(struct stmmac_priv *priv)
	1790	{
	1791	u32 tx_count = priv->plat->tx_queues_to_use;
	1792	int ret = -ENOMEM;
	1793	u32 queue;
	1794
	1795	/* TX queues buffers and DMA */
	1796	for (queue = 0; queue < tx_count; queue++) {
	1797	struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
	1798	size_t size;
	1799	void *addr;
	1800
	1801	tx_q->queue_index = queue;
	1802	tx_q->priv_data = priv;
	1803
	1804	tx_q->tx_skbuff_dma = kcalloc(priv->dma_tx_size,
	1805	sizeof(*tx_q->tx_skbuff_dma),
	1806	GFP_KERNEL);
	1807	if (!tx_q->tx_skbuff_dma)
	1808	goto err_dma;
	1809
	1810	tx_q->tx_skbuff = kcalloc(priv->dma_tx_size,
	1811	sizeof(struct sk_buff *),
	1812	GFP_KERNEL);
	1813	if (!tx_q->tx_skbuff)
	1814	goto err_dma;
	1815
	1816	if (priv->extend_desc)
	1817	size = sizeof(struct dma_extended_desc);
	1818	else if (tx_q->tbs & STMMAC_TBS_AVAIL)
	1819	size = sizeof(struct dma_edesc);
	1820	else
	1821	size = sizeof(struct dma_desc);
	1822
	1823	size *= priv->dma_tx_size;
	1824
	1825	addr = dma_alloc_coherent(priv->device, size,
	1826	&tx_q->dma_tx_phy, GFP_KERNEL);
	1827	if (!addr)
	1828	goto err_dma;
	1829
	1830	if (priv->extend_desc)
	1831	tx_q->dma_etx = addr;
	1832	else if (tx_q->tbs & STMMAC_TBS_AVAIL)
	1833	tx_q->dma_entx = addr;
	1834	else
	1835	tx_q->dma_tx = addr;
	1836	}
	1837
	1838	return 0;
	1839
	1840	err_dma:
	1841	free_dma_tx_desc_resources(priv);
	1842	return ret;
	1843	}
	1844
	1845	/**
	1846	* alloc_dma_desc_resources - alloc TX/RX resources.
	1847	* @priv: private structure
	1848	* Description: according to which descriptor can be used (extend or basic)
	1849	* this function allocates the resources for TX and RX paths. In case of
	1850	* reception, for example, it pre-allocated the RX socket buffer in order to
	1851	* allow zero-copy mechanism.
	1852	*/
	1853	static int alloc_dma_desc_resources(struct stmmac_priv *priv)
	1854	{
	1855	/* RX Allocation */
	1856	int ret = alloc_dma_rx_desc_resources(priv);
	1857
	1858	if (ret)
	1859	return ret;
	1860
	1861	ret = alloc_dma_tx_desc_resources(priv);
	1862
	1863	return ret;
	1864	}
	1865
	1866	/**
	1867	* free_dma_desc_resources - free dma desc resources
	1868	* @priv: private structure
	1869	*/
	1870	static void free_dma_desc_resources(struct stmmac_priv *priv)
	1871	{
	1872	/* Release the DMA RX socket buffers */
	1873	free_dma_rx_desc_resources(priv);
	1874
	1875	/* Release the DMA TX socket buffers */
	1876	free_dma_tx_desc_resources(priv);
	1877	}
	1878
	1879	/**
	1880	* stmmac_mac_enable_rx_queues - Enable MAC rx queues
	1881	* @priv: driver private structure
	1882	* Description: It is used for enabling the rx queues in the MAC
	1883	*/
	1884	static void stmmac_mac_enable_rx_queues(struct stmmac_priv *priv)
	1885	{
	1886	u32 rx_queues_count = priv->plat->rx_queues_to_use;
	1887	int queue;
	1888	u8 mode;
	1889
	1890	for (queue = 0; queue < rx_queues_count; queue++) {
	1891	mode = priv->plat->rx_queues_cfg[queue].mode_to_use;
	1892	stmmac_rx_queue_enable(priv, priv->hw, mode, queue);
	1893	}
	1894	}
	1895
	1896	/**
	1897	* stmmac_start_rx_dma - start RX DMA channel
	1898	* @priv: driver private structure
	1899	* @chan: RX channel index
	1900	* Description:
	1901	* This starts a RX DMA channel
	1902	*/
	1903	static void stmmac_start_rx_dma(struct stmmac_priv *priv, u32 chan)
	1904	{
	1905	netdev_dbg(priv->dev, "DMA RX processes started in channel %d\n", chan);
	1906	stmmac_start_rx(priv, priv->ioaddr, chan);
	1907	}
	1908
	1909	/**
	1910	* stmmac_start_tx_dma - start TX DMA channel
	1911	* @priv: driver private structure
	1912	* @chan: TX channel index
	1913	* Description:
	1914	* This starts a TX DMA channel
	1915	*/
	1916	static void stmmac_start_tx_dma(struct stmmac_priv *priv, u32 chan)
	1917	{
	1918	netdev_dbg(priv->dev, "DMA TX processes started in channel %d\n", chan);
	1919	stmmac_start_tx(priv, priv->ioaddr, chan);
	1920	}
	1921
	1922	/**
	1923	* stmmac_stop_rx_dma - stop RX DMA channel
	1924	* @priv: driver private structure
	1925	* @chan: RX channel index
	1926	* Description:
	1927	* This stops a RX DMA channel
	1928	*/
	1929	static void stmmac_stop_rx_dma(struct stmmac_priv *priv, u32 chan)
	1930	{
	1931	netdev_dbg(priv->dev, "DMA RX processes stopped in channel %d\n", chan);
	1932	stmmac_stop_rx(priv, priv->ioaddr, chan);
	1933	}
	1934
	1935	/**
	1936	* stmmac_stop_tx_dma - stop TX DMA channel
	1937	* @priv: driver private structure
	1938	* @chan: TX channel index
	1939	* Description:
	1940	* This stops a TX DMA channel
	1941	*/
	1942	static void stmmac_stop_tx_dma(struct stmmac_priv *priv, u32 chan)
	1943	{
	1944	netdev_dbg(priv->dev, "DMA TX processes stopped in channel %d\n", chan);
	1945	stmmac_stop_tx(priv, priv->ioaddr, chan);
	1946	}
	1947
	1948	/**
	1949	* stmmac_start_all_dma - start all RX and TX DMA channels
	1950	* @priv: driver private structure
	1951	* Description:
	1952	* This starts all the RX and TX DMA channels
	1953	*/
	1954	static void stmmac_start_all_dma(struct stmmac_priv *priv)
	1955	{
	1956	u32 rx_channels_count = priv->plat->rx_queues_to_use;
	1957	u32 tx_channels_count = priv->plat->tx_queues_to_use;
	1958	u32 chan = 0;
	1959
	1960	for (chan = 0; chan < rx_channels_count; chan++)
	1961	stmmac_start_rx_dma(priv, chan);
	1962
	1963	for (chan = 0; chan < tx_channels_count; chan++)
	1964	stmmac_start_tx_dma(priv, chan);
	1965	}
	1966
	1967	/**
	1968	* stmmac_stop_all_dma - stop all RX and TX DMA channels
	1969	* @priv: driver private structure
	1970	* Description:
	1971	* This stops the RX and TX DMA channels
	1972	*/
	1973	static void stmmac_stop_all_dma(struct stmmac_priv *priv)
	1974	{
	1975	u32 rx_channels_count = priv->plat->rx_queues_to_use;
	1976	u32 tx_channels_count = priv->plat->tx_queues_to_use;
	1977	u32 chan = 0;
	1978
	1979	for (chan = 0; chan < rx_channels_count; chan++)
	1980	stmmac_stop_rx_dma(priv, chan);
	1981
	1982	for (chan = 0; chan < tx_channels_count; chan++)
	1983	stmmac_stop_tx_dma(priv, chan);
	1984	}
	1985
	1986	/**
	1987	* stmmac_dma_operation_mode - HW DMA operation mode
	1988	* @priv: driver private structure
	1989	* Description: it is used for configuring the DMA operation mode register in
	1990	* order to program the tx/rx DMA thresholds or Store-And-Forward mode.
	1991	*/
	1992	static void stmmac_dma_operation_mode(struct stmmac_priv *priv)
	1993	{
	1994	u32 rx_channels_count = priv->plat->rx_queues_to_use;
	1995	u32 tx_channels_count = priv->plat->tx_queues_to_use;
	1996	int rxfifosz = priv->plat->rx_fifo_size;
	1997	int txfifosz = priv->plat->tx_fifo_size;
	1998	u32 txmode = 0;
	1999	u32 rxmode = 0;
	2000	u32 chan = 0;
	2001	u8 qmode = 0;
	2002
	2003	if (rxfifosz == 0)
	2004	rxfifosz = priv->dma_cap.rx_fifo_size;
	2005	if (txfifosz == 0)
	2006	txfifosz = priv->dma_cap.tx_fifo_size;
	2007
	2008	/* Adjust for real per queue fifo size */
	2009	rxfifosz /= rx_channels_count;
	2010	txfifosz /= tx_channels_count;
	2011
	2012	if (priv->plat->force_thresh_dma_mode) {
	2013	txmode = tc;
	2014	rxmode = tc;
	2015	} else if (priv->plat->force_sf_dma_mode \|\| priv->plat->tx_coe) {
	2016	/*
	2017	* In case of GMAC, SF mode can be enabled
	2018	* to perform the TX COE in HW. This depends on:
	2019	* 1) TX COE if actually supported
	2020	* 2) There is no bugged Jumbo frame support
	2021	* that needs to not insert csum in the TDES.
	2022	*/
	2023	txmode = SF_DMA_MODE;
	2024	rxmode = SF_DMA_MODE;
	2025	priv->xstats.threshold = SF_DMA_MODE;
	2026	} else {
	2027	txmode = tc;
	2028	rxmode = SF_DMA_MODE;
	2029	}
	2030
	2031	/* configure all channels */
	2032	for (chan = 0; chan < rx_channels_count; chan++) {
	2033	qmode = priv->plat->rx_queues_cfg[chan].mode_to_use;
	2034
	2035	stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan,
	2036	rxfifosz, qmode);
	2037	stmmac_set_dma_bfsize(priv, priv->ioaddr, priv->dma_buf_sz,
	2038	chan);
	2039	}
	2040
	2041	for (chan = 0; chan < tx_channels_count; chan++) {
	2042	qmode = priv->plat->tx_queues_cfg[chan].mode_to_use;
	2043
	2044	stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan,
	2045	txfifosz, qmode);
	2046	}
	2047	}
	2048
	2049	/**
	2050	* stmmac_tx_clean - to manage the transmission completion
	2051	* @priv: driver private structure
	2052	* @budget: napi budget limiting this functions packet handling
	2053	* @queue: TX queue index
	2054	* Description: it reclaims the transmit resources after transmission completes.
	2055	*/
	2056	static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue)
	2057	{
	2058	struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
	2059	unsigned int bytes_compl = 0, pkts_compl = 0;
	2060	unsigned int entry, count = 0;
	2061
	2062	__netif_tx_lock_bh(netdev_get_tx_queue(priv->dev, queue));
	2063
	2064	priv->xstats.tx_clean++;
	2065
	2066	entry = tx_q->dirty_tx;
	2067	while ((entry != tx_q->cur_tx) && (count < budget)) {
	2068	struct sk_buff *skb = tx_q->tx_skbuff[entry];
	2069	struct dma_desc *p;
	2070	int status;
	2071
	2072	if (priv->extend_desc)
	2073	p = (struct dma_desc *)(tx_q->dma_etx + entry);
	2074	else if (tx_q->tbs & STMMAC_TBS_AVAIL)
	2075	p = &tx_q->dma_entx[entry].basic;
	2076	else
	2077	p = tx_q->dma_tx + entry;
	2078
	2079	status = stmmac_tx_status(priv, &priv->dev->stats,
	2080	&priv->xstats, p, priv->ioaddr);
	2081	/* Check if the descriptor is owned by the DMA */
	2082	if (unlikely(status & tx_dma_own))
	2083	break;
	2084
	2085	count++;
	2086
	2087	/* Make sure descriptor fields are read after reading
	2088	* the own bit.
	2089	*/
	2090	dma_rmb();
	2091
	2092	/* Just consider the last segment and ...*/
	2093	if (likely(!(status & tx_not_ls))) {
	2094	/* ... verify the status error condition */
	2095	if (unlikely(status & tx_err)) {
	2096	priv->dev->stats.tx_errors++;
	2097	} else {
	2098	priv->dev->stats.tx_packets++;
	2099	priv->xstats.tx_pkt_n++;
	2100	}
	2101	stmmac_get_tx_hwtstamp(priv, p, skb);
	2102	}
	2103
	2104	if (likely(tx_q->tx_skbuff_dma[entry].buf)) {
	2105	if (tx_q->tx_skbuff_dma[entry].map_as_page)
	2106	dma_unmap_page(priv->device,
	2107	tx_q->tx_skbuff_dma[entry].buf,
	2108	tx_q->tx_skbuff_dma[entry].len,
	2109	DMA_TO_DEVICE);
	2110	else
	2111	dma_unmap_single(priv->device,
	2112	tx_q->tx_skbuff_dma[entry].buf,
	2113	tx_q->tx_skbuff_dma[entry].len,
	2114	DMA_TO_DEVICE);
	2115	tx_q->tx_skbuff_dma[entry].buf = 0;
	2116	tx_q->tx_skbuff_dma[entry].len = 0;
	2117	tx_q->tx_skbuff_dma[entry].map_as_page = false;
	2118	}
	2119
	2120	stmmac_clean_desc3(priv, tx_q, p);
	2121
	2122	tx_q->tx_skbuff_dma[entry].last_segment = false;
	2123	tx_q->tx_skbuff_dma[entry].is_jumbo = false;
	2124
	2125	if (likely(skb != NULL)) {
	2126	pkts_compl++;
	2127	bytes_compl += skb->len;
	2128	dev_consume_skb_any(skb);
	2129	tx_q->tx_skbuff[entry] = NULL;
	2130	}
	2131
	2132	stmmac_release_tx_desc(priv, p, priv->mode);
	2133
	2134	entry = STMMAC_GET_ENTRY(entry, priv->dma_tx_size);
	2135	}
	2136	tx_q->dirty_tx = entry;
	2137
	2138	netdev_tx_completed_queue(netdev_get_tx_queue(priv->dev, queue),
	2139	pkts_compl, bytes_compl);
	2140
	2141	if (unlikely(netif_tx_queue_stopped(netdev_get_tx_queue(priv->dev,
	2142	queue))) &&
	2143	stmmac_tx_avail(priv, queue) > STMMAC_TX_THRESH(priv)) {
	2144
	2145	netif_dbg(priv, tx_done, priv->dev,
	2146	"%s: restart transmit\n", __func__);
	2147	netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, queue));
	2148	}
	2149
	2150	if ((priv->eee_enabled) && (!priv->tx_path_in_lpi_mode)) {
	2151	stmmac_enable_eee_mode(priv);
	2152	mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer));
	2153	}
	2154
	2155	/* We still have pending packets, let's call for a new scheduling */
	2156	if (tx_q->dirty_tx != tx_q->cur_tx)
	2157	mod_timer(&tx_q->txtimer, STMMAC_COAL_TIMER(priv->tx_coal_timer));
	2158
	2159	__netif_tx_unlock_bh(netdev_get_tx_queue(priv->dev, queue));
	2160
	2161	return count;
	2162	}
	2163
	2164	/**
	2165	* stmmac_tx_err - to manage the tx error
	2166	* @priv: driver private structure
	2167	* @chan: channel index
	2168	* Description: it cleans the descriptors and restarts the transmission
	2169	* in case of transmission errors.
	2170	*/
	2171	static void stmmac_tx_err(struct stmmac_priv *priv, u32 chan)
	2172	{
	2173	struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan];
	2174
	2175	netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, chan));
	2176
	2177	stmmac_stop_tx_dma(priv, chan);
	2178	dma_free_tx_skbufs(priv, chan);
	2179	stmmac_clear_tx_descriptors(priv, chan);
	2180	tx_q->dirty_tx = 0;
	2181	tx_q->cur_tx = 0;
	2182	tx_q->mss = 0;
	2183	netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, chan));
	2184	stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
	2185	tx_q->dma_tx_phy, chan);
	2186	stmmac_start_tx_dma(priv, chan);
	2187
	2188	priv->dev->stats.tx_errors++;
	2189	netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, chan));
	2190	}
	2191
	2192	/**
	2193	* stmmac_set_dma_operation_mode - Set DMA operation mode by channel
	2194	* @priv: driver private structure
	2195	* @txmode: TX operating mode
	2196	* @rxmode: RX operating mode
	2197	* @chan: channel index
	2198	* Description: it is used for configuring of the DMA operation mode in
	2199	* runtime in order to program the tx/rx DMA thresholds or Store-And-Forward
	2200	* mode.
	2201	*/
	2202	static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode,
	2203	u32 rxmode, u32 chan)
	2204	{
	2205	u8 rxqmode = priv->plat->rx_queues_cfg[chan].mode_to_use;
	2206	u8 txqmode = priv->plat->tx_queues_cfg[chan].mode_to_use;
	2207	u32 rx_channels_count = priv->plat->rx_queues_to_use;
	2208	u32 tx_channels_count = priv->plat->tx_queues_to_use;
	2209	int rxfifosz = priv->plat->rx_fifo_size;
	2210	int txfifosz = priv->plat->tx_fifo_size;
	2211
	2212	if (rxfifosz == 0)
	2213	rxfifosz = priv->dma_cap.rx_fifo_size;
	2214	if (txfifosz == 0)
	2215	txfifosz = priv->dma_cap.tx_fifo_size;
	2216
	2217	/* Adjust for real per queue fifo size */
	2218	rxfifosz /= rx_channels_count;
	2219	txfifosz /= tx_channels_count;
	2220
	2221	stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, rxfifosz, rxqmode);
	2222	stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, txfifosz, txqmode);
	2223	}
	2224
	2225	static bool stmmac_safety_feat_interrupt(struct stmmac_priv *priv)
	2226	{
	2227	int ret;
	2228
	2229	ret = stmmac_safety_feat_irq_status(priv, priv->dev,
	2230	priv->ioaddr, priv->dma_cap.asp, &priv->sstats);
	2231	if (ret && (ret != -EINVAL)) {
	2232	stmmac_global_err(priv);
	2233	return true;
	2234	}
	2235
	2236	return false;
	2237	}
	2238
	2239	static int stmmac_napi_check(struct stmmac_priv *priv, u32 chan)
	2240	{
	2241	int status = stmmac_dma_interrupt_status(priv, priv->ioaddr,
	2242	&priv->xstats, chan);
	2243	struct stmmac_channel *ch = &priv->channel[chan];
	2244	unsigned long flags;
	2245
	2246	if ((status & handle_rx) && (chan < priv->plat->rx_queues_to_use)) {
	2247	if (napi_schedule_prep(&ch->rx_napi)) {
	2248	spin_lock_irqsave(&ch->lock, flags);
	2249	stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 0);
	2250	spin_unlock_irqrestore(&ch->lock, flags);
	2251	__napi_schedule(&ch->rx_napi);
	2252	}
	2253	}
	2254
	2255	if ((status & handle_tx) && (chan < priv->plat->tx_queues_to_use)) {
	2256	if (napi_schedule_prep(&ch->tx_napi)) {
	2257	spin_lock_irqsave(&ch->lock, flags);
	2258	stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 0, 1);
	2259	spin_unlock_irqrestore(&ch->lock, flags);
	2260	__napi_schedule(&ch->tx_napi);
	2261	}
	2262	}
	2263
	2264	return status;
	2265	}
	2266
	2267	/**
	2268	* stmmac_dma_interrupt - DMA ISR
	2269	* @priv: driver private structure
	2270	* Description: this is the DMA ISR. It is called by the main ISR.
	2271	* It calls the dwmac dma routine and schedule poll method in case of some
	2272	* work can be done.
	2273	*/
	2274	static void stmmac_dma_interrupt(struct stmmac_priv *priv)
	2275	{
	2276	u32 tx_channel_count = priv->plat->tx_queues_to_use;
	2277	u32 rx_channel_count = priv->plat->rx_queues_to_use;
	2278	u32 channels_to_check = tx_channel_count > rx_channel_count ?
	2279	tx_channel_count : rx_channel_count;
	2280	u32 chan;
	2281	int status[max_t(u32, MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES)];
	2282
	2283	/* Make sure we never check beyond our status buffer. */
	2284	if (WARN_ON_ONCE(channels_to_check > ARRAY_SIZE(status)))
	2285	channels_to_check = ARRAY_SIZE(status);
	2286
	2287	for (chan = 0; chan < channels_to_check; chan++)
	2288	status[chan] = stmmac_napi_check(priv, chan);
	2289
	2290	for (chan = 0; chan < tx_channel_count; chan++) {
	2291	if (unlikely(status[chan] & tx_hard_error_bump_tc)) {
	2292	/* Try to bump up the dma threshold on this failure */
	2293	if (unlikely(priv->xstats.threshold != SF_DMA_MODE) &&
	2294	(tc <= 256)) {
	2295	tc += 64;
	2296	if (priv->plat->force_thresh_dma_mode)
	2297	stmmac_set_dma_operation_mode(priv,
	2298	tc,
	2299	tc,
	2300	chan);
	2301	else
	2302	stmmac_set_dma_operation_mode(priv,
	2303	tc,
	2304	SF_DMA_MODE,
	2305	chan);
	2306	priv->xstats.threshold = tc;
	2307	}
	2308	} else if (unlikely(status[chan] == tx_hard_error)) {
	2309	stmmac_tx_err(priv, chan);
	2310	}
	2311	}
	2312	}
	2313
	2314	/**
	2315	* stmmac_mmc_setup: setup the Mac Management Counters (MMC)
	2316	* @priv: driver private structure
	2317	* Description: this masks the MMC irq, in fact, the counters are managed in SW.
	2318	*/
	2319	static void stmmac_mmc_setup(struct stmmac_priv *priv)
	2320	{
	2321	unsigned int mode = MMC_CNTRL_RESET_ON_READ \| MMC_CNTRL_COUNTER_RESET \|
	2322	MMC_CNTRL_PRESET \| MMC_CNTRL_FULL_HALF_PRESET;
	2323
	2324	stmmac_mmc_intr_all_mask(priv, priv->mmcaddr);
	2325
	2326	if (priv->dma_cap.rmon) {
	2327	stmmac_mmc_ctrl(priv, priv->mmcaddr, mode);
	2328	memset(&priv->mmc, 0, sizeof(struct stmmac_counters));
	2329	} else
	2330	netdev_info(priv->dev, "No MAC Management Counters available\n");
	2331	}
	2332
	2333	/**
	2334	* stmmac_get_hw_features - get MAC capabilities from the HW cap. register.
	2335	* @priv: driver private structure
	2336	* Description:
	2337	* new GMAC chip generations have a new register to indicate the
	2338	* presence of the optional feature/functions.
	2339	* This can be also used to override the value passed through the
	2340	* platform and necessary for old MAC10/100 and GMAC chips.
	2341	*/
	2342	static int stmmac_get_hw_features(struct stmmac_priv *priv)
	2343	{
	2344	return stmmac_get_hw_feature(priv, priv->ioaddr, &priv->dma_cap) == 0;
	2345	}
	2346
	2347	/**
	2348	* stmmac_check_ether_addr - check if the MAC addr is valid
	2349	* @priv: driver private structure
	2350	* Description:
	2351	* it is to verify if the MAC address is valid, in case of failures it
	2352	* generates a random MAC address
	2353	*/
	2354	static void stmmac_check_ether_addr(struct stmmac_priv *priv)
	2355	{
7e970c	2356	// if (!is_valid_ether_addr(priv->dev->dev_addr)) {
H	2357	if(1) {
a07526	2358	stmmac_get_umac_addr(priv, priv->hw, priv->dev->dev_addr, 0);
H	2359	if (likely(priv->plat->get_eth_addr))
	2360	priv->plat->get_eth_addr(priv->plat->bsp_priv,
	2361	priv->dev->dev_addr);
	2362	if (!is_valid_ether_addr(priv->dev->dev_addr))
	2363	eth_hw_addr_random(priv->dev);
	2364	dev_info(priv->device, "device MAC address %pM\n",
	2365	priv->dev->dev_addr);
	2366	}
	2367	}
	2368
	2369	/**
	2370	* stmmac_init_dma_engine - DMA init.
	2371	* @priv: driver private structure
	2372	* Description:
	2373	* It inits the DMA invoking the specific MAC/GMAC callback.
	2374	* Some DMA parameters can be passed from the platform;
	2375	* in case of these are not passed a default is kept for the MAC or GMAC.
	2376	*/
	2377	static int stmmac_init_dma_engine(struct stmmac_priv *priv)
	2378	{
	2379	u32 rx_channels_count = priv->plat->rx_queues_to_use;
	2380	u32 tx_channels_count = priv->plat->tx_queues_to_use;
	2381	u32 dma_csr_ch = max(rx_channels_count, tx_channels_count);
	2382	struct stmmac_rx_queue *rx_q;
	2383	struct stmmac_tx_queue *tx_q;
	2384	u32 chan = 0;
	2385	int atds = 0;
	2386	int ret = 0;
	2387
	2388	if (!priv->plat->dma_cfg \|\| !priv->plat->dma_cfg->pbl) {
	2389	dev_err(priv->device, "Invalid DMA configuration\n");
	2390	return -EINVAL;
	2391	}
	2392
	2393	if (priv->extend_desc && (priv->mode == STMMAC_RING_MODE))
	2394	atds = 1;
	2395
	2396	ret = stmmac_reset(priv, priv->ioaddr);
	2397	if (ret) {
	2398	dev_err(priv->device, "Failed to reset the dma\n");
	2399	return ret;
	2400	}
	2401
	2402	/* DMA Configuration */
	2403	stmmac_dma_init(priv, priv->ioaddr, priv->plat->dma_cfg, atds);
	2404
	2405	if (priv->plat->axi)
	2406	stmmac_axi(priv, priv->ioaddr, priv->plat->axi);
	2407
	2408	/* DMA CSR Channel configuration */
	2409	for (chan = 0; chan < dma_csr_ch; chan++)
	2410	stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan);
	2411
	2412	/* DMA RX Channel Configuration */
	2413	for (chan = 0; chan < rx_channels_count; chan++) {
	2414	rx_q = &priv->rx_queue[chan];
	2415
	2416	stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
	2417	rx_q->dma_rx_phy, chan);
	2418
	2419	rx_q->rx_tail_addr = rx_q->dma_rx_phy +
	2420	(priv->dma_rx_size *
	2421	sizeof(struct dma_desc));
	2422	stmmac_set_rx_tail_ptr(priv, priv->ioaddr,
	2423	rx_q->rx_tail_addr, chan);
	2424	}
	2425
	2426	/* DMA TX Channel Configuration */
	2427	for (chan = 0; chan < tx_channels_count; chan++) {
	2428	tx_q = &priv->tx_queue[chan];
	2429
	2430	stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
	2431	tx_q->dma_tx_phy, chan);
	2432
	2433	tx_q->tx_tail_addr = tx_q->dma_tx_phy;
	2434	stmmac_set_tx_tail_ptr(priv, priv->ioaddr,
	2435	tx_q->tx_tail_addr, chan);
	2436	}
	2437
	2438	return ret;
	2439	}
	2440
	2441	static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue)
	2442	{
	2443	struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
	2444
	2445	mod_timer(&tx_q->txtimer, STMMAC_COAL_TIMER(priv->tx_coal_timer));
	2446	}
	2447
	2448	/**
	2449	* stmmac_tx_timer - mitigation sw timer for tx.
	2450	* @t: data pointer
	2451	* Description:
	2452	* This is the timer handler to directly invoke the stmmac_tx_clean.
	2453	*/
	2454	static void stmmac_tx_timer(struct timer_list *t)
	2455	{
	2456	struct stmmac_tx_queue *tx_q = from_timer(tx_q, t, txtimer);
	2457	struct stmmac_priv *priv = tx_q->priv_data;
	2458	struct stmmac_channel *ch;
	2459
	2460	ch = &priv->channel[tx_q->queue_index];
	2461
	2462	if (likely(napi_schedule_prep(&ch->tx_napi))) {
	2463	unsigned long flags;
	2464
	2465	spin_lock_irqsave(&ch->lock, flags);
	2466	stmmac_disable_dma_irq(priv, priv->ioaddr, ch->index, 0, 1);
	2467	spin_unlock_irqrestore(&ch->lock, flags);
	2468	__napi_schedule(&ch->tx_napi);
	2469	}
	2470	}
	2471
	2472	/**
	2473	* stmmac_init_coalesce - init mitigation options.
	2474	* @priv: driver private structure
	2475	* Description:
	2476	* This inits the coalesce parameters: i.e. timer rate,
	2477	* timer handler and default threshold used for enabling the
	2478	* interrupt on completion bit.
	2479	*/
	2480	static void stmmac_init_coalesce(struct stmmac_priv *priv)
	2481	{
	2482	u32 tx_channel_count = priv->plat->tx_queues_to_use;
	2483	u32 chan;
	2484
	2485	priv->tx_coal_frames = STMMAC_TX_FRAMES;
	2486	priv->tx_coal_timer = STMMAC_COAL_TX_TIMER;
	2487	priv->rx_coal_frames = STMMAC_RX_FRAMES;
	2488
	2489	for (chan = 0; chan < tx_channel_count; chan++) {
	2490	struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan];
	2491
	2492	timer_setup(&tx_q->txtimer, stmmac_tx_timer, 0);
	2493	}
	2494	}
	2495
	2496	static void stmmac_set_rings_length(struct stmmac_priv *priv)
	2497	{
	2498	u32 rx_channels_count = priv->plat->rx_queues_to_use;
	2499	u32 tx_channels_count = priv->plat->tx_queues_to_use;
	2500	u32 chan;
	2501
	2502	/* set TX ring length */
	2503	for (chan = 0; chan < tx_channels_count; chan++)
	2504	stmmac_set_tx_ring_len(priv, priv->ioaddr,
	2505	(priv->dma_tx_size - 1), chan);
	2506
	2507	/* set RX ring length */
	2508	for (chan = 0; chan < rx_channels_count; chan++)
	2509	stmmac_set_rx_ring_len(priv, priv->ioaddr,
	2510	(priv->dma_rx_size - 1), chan);
	2511	}
	2512
	2513	/**
	2514	* stmmac_set_tx_queue_weight - Set TX queue weight
	2515	* @priv: driver private structure
	2516	* Description: It is used for setting TX queues weight
	2517	*/
	2518	static void stmmac_set_tx_queue_weight(struct stmmac_priv *priv)
	2519	{
	2520	u32 tx_queues_count = priv->plat->tx_queues_to_use;
	2521	u32 weight;
	2522	u32 queue;
	2523
	2524	for (queue = 0; queue < tx_queues_count; queue++) {
	2525	weight = priv->plat->tx_queues_cfg[queue].weight;
	2526	stmmac_set_mtl_tx_queue_weight(priv, priv->hw, weight, queue);
	2527	}
	2528	}
	2529
	2530	/**
	2531	* stmmac_configure_cbs - Configure CBS in TX queue
	2532	* @priv: driver private structure
	2533	* Description: It is used for configuring CBS in AVB TX queues
	2534	*/
	2535	static void stmmac_configure_cbs(struct stmmac_priv *priv)
	2536	{
	2537	u32 tx_queues_count = priv->plat->tx_queues_to_use;
	2538	u32 mode_to_use;
	2539	u32 queue;
	2540
	2541	/* queue 0 is reserved for legacy traffic */
	2542	for (queue = 1; queue < tx_queues_count; queue++) {
	2543	mode_to_use = priv->plat->tx_queues_cfg[queue].mode_to_use;
	2544	if (mode_to_use == MTL_QUEUE_DCB)
	2545	continue;
	2546
	2547	stmmac_config_cbs(priv, priv->hw,
	2548	priv->plat->tx_queues_cfg[queue].send_slope,
	2549	priv->plat->tx_queues_cfg[queue].idle_slope,
	2550	priv->plat->tx_queues_cfg[queue].high_credit,
	2551	priv->plat->tx_queues_cfg[queue].low_credit,
	2552	queue);
	2553	}
	2554	}
	2555
	2556	/**
	2557	* stmmac_rx_queue_dma_chan_map - Map RX queue to RX dma channel
	2558	* @priv: driver private structure
	2559	* Description: It is used for mapping RX queues to RX dma channels
	2560	*/
	2561	static void stmmac_rx_queue_dma_chan_map(struct stmmac_priv *priv)
	2562	{
	2563	u32 rx_queues_count = priv->plat->rx_queues_to_use;
	2564	u32 queue;
	2565	u32 chan;
	2566
	2567	for (queue = 0; queue < rx_queues_count; queue++) {
	2568	chan = priv->plat->rx_queues_cfg[queue].chan;
	2569	stmmac_map_mtl_to_dma(priv, priv->hw, queue, chan);
	2570	}
	2571	}
	2572
	2573	/**
	2574	* stmmac_mac_config_rx_queues_prio - Configure RX Queue priority
	2575	* @priv: driver private structure
	2576	* Description: It is used for configuring the RX Queue Priority
	2577	*/
	2578	static void stmmac_mac_config_rx_queues_prio(struct stmmac_priv *priv)
	2579	{
	2580	u32 rx_queues_count = priv->plat->rx_queues_to_use;
	2581	u32 queue;
	2582	u32 prio;
	2583
	2584	for (queue = 0; queue < rx_queues_count; queue++) {
	2585	if (!priv->plat->rx_queues_cfg[queue].use_prio)
	2586	continue;
	2587
	2588	prio = priv->plat->rx_queues_cfg[queue].prio;
	2589	stmmac_rx_queue_prio(priv, priv->hw, prio, queue);
	2590	}
	2591	}
	2592
	2593	/**
	2594	* stmmac_mac_config_tx_queues_prio - Configure TX Queue priority
	2595	* @priv: driver private structure
	2596	* Description: It is used for configuring the TX Queue Priority
	2597	*/
	2598	static void stmmac_mac_config_tx_queues_prio(struct stmmac_priv *priv)
	2599	{
	2600	u32 tx_queues_count = priv->plat->tx_queues_to_use;
	2601	u32 queue;
	2602	u32 prio;
	2603
	2604	for (queue = 0; queue < tx_queues_count; queue++) {
	2605	if (!priv->plat->tx_queues_cfg[queue].use_prio)
	2606	continue;
	2607
	2608	prio = priv->plat->tx_queues_cfg[queue].prio;
	2609	stmmac_tx_queue_prio(priv, priv->hw, prio, queue);
	2610	}
	2611	}
	2612
	2613	/**
	2614	* stmmac_mac_config_rx_queues_routing - Configure RX Queue Routing
	2615	* @priv: driver private structure
	2616	* Description: It is used for configuring the RX queue routing
	2617	*/
	2618	static void stmmac_mac_config_rx_queues_routing(struct stmmac_priv *priv)
	2619	{
	2620	u32 rx_queues_count = priv->plat->rx_queues_to_use;
	2621	u32 queue;
	2622	u8 packet;
	2623
	2624	for (queue = 0; queue < rx_queues_count; queue++) {
	2625	/* no specific packet type routing specified for the queue */
	2626	if (priv->plat->rx_queues_cfg[queue].pkt_route == 0x0)
	2627	continue;
	2628
	2629	packet = priv->plat->rx_queues_cfg[queue].pkt_route;
	2630	stmmac_rx_queue_routing(priv, priv->hw, packet, queue);
	2631	}
	2632	}
	2633
	2634	static void stmmac_mac_config_rss(struct stmmac_priv *priv)
	2635	{
	2636	if (!priv->dma_cap.rssen \|\| !priv->plat->rss_en) {
	2637	priv->rss.enable = false;
	2638	return;
	2639	}
	2640
	2641	if (priv->dev->features & NETIF_F_RXHASH)
	2642	priv->rss.enable = true;
	2643	else
	2644	priv->rss.enable = false;
	2645
	2646	stmmac_rss_configure(priv, priv->hw, &priv->rss,
	2647	priv->plat->rx_queues_to_use);
	2648	}
	2649
	2650	/**
	2651	* stmmac_mtl_configuration - Configure MTL
	2652	* @priv: driver private structure
	2653	* Description: It is used for configurring MTL
	2654	*/
	2655	static void stmmac_mtl_configuration(struct stmmac_priv *priv)
	2656	{
	2657	u32 rx_queues_count = priv->plat->rx_queues_to_use;
	2658	u32 tx_queues_count = priv->plat->tx_queues_to_use;
	2659
	2660	if (tx_queues_count > 1)
	2661	stmmac_set_tx_queue_weight(priv);
	2662
	2663	/* Configure MTL RX algorithms */
	2664	if (rx_queues_count > 1)
	2665	stmmac_prog_mtl_rx_algorithms(priv, priv->hw,
	2666	priv->plat->rx_sched_algorithm);
	2667
	2668	/* Configure MTL TX algorithms */
	2669	if (tx_queues_count > 1)
	2670	stmmac_prog_mtl_tx_algorithms(priv, priv->hw,
	2671	priv->plat->tx_sched_algorithm);
	2672
	2673	/* Configure CBS in AVB TX queues */
	2674	if (tx_queues_count > 1)
	2675	stmmac_configure_cbs(priv);
	2676
	2677	/* Map RX MTL to DMA channels */
	2678	stmmac_rx_queue_dma_chan_map(priv);
	2679
	2680	/* Enable MAC RX Queues */
	2681	stmmac_mac_enable_rx_queues(priv);
	2682
	2683	/* Set RX priorities */
	2684	if (rx_queues_count > 1)
	2685	stmmac_mac_config_rx_queues_prio(priv);
	2686
	2687	/* Set TX priorities */
	2688	if (tx_queues_count > 1)
	2689	stmmac_mac_config_tx_queues_prio(priv);
	2690
	2691	/* Set RX routing */
	2692	if (rx_queues_count > 1)
	2693	stmmac_mac_config_rx_queues_routing(priv);
	2694
	2695	/* Receive Side Scaling */
	2696	if (rx_queues_count > 1)
	2697	stmmac_mac_config_rss(priv);
	2698	}
	2699
	2700	static void stmmac_safety_feat_configuration(struct stmmac_priv *priv)
	2701	{
	2702	if (priv->dma_cap.asp) {
	2703	netdev_info(priv->dev, "Enabling Safety Features\n");
	2704	stmmac_safety_feat_config(priv, priv->ioaddr, priv->dma_cap.asp);
	2705	} else {
	2706	netdev_info(priv->dev, "No Safety Features support found\n");
	2707	}
	2708	}
	2709
	2710	/**
	2711	* stmmac_hw_setup - setup mac in a usable state.
	2712	* @dev : pointer to the device structure.
	2713	* @ptp_register: register PTP if set
	2714	* Description:
	2715	* this is the main function to setup the HW in a usable state because the
	2716	* dma engine is reset, the core registers are configured (e.g. AXI,
	2717	* Checksum features, timers). The DMA is ready to start receiving and
	2718	* transmitting.
	2719	* Return value:
	2720	* 0 on success and an appropriate (-)ve integer as defined in errno.h
	2721	* file on failure.
	2722	*/
	2723	static int stmmac_hw_setup(struct net_device *dev, bool ptp_register)
	2724	{
	2725	struct stmmac_priv *priv = netdev_priv(dev);
	2726	u32 rx_cnt = priv->plat->rx_queues_to_use;
	2727	u32 tx_cnt = priv->plat->tx_queues_to_use;
	2728	u32 chan;
	2729	int ret;
	2730
	2731	/* DMA initialization and SW reset */
	2732	ret = stmmac_init_dma_engine(priv);
	2733	if (ret < 0) {
	2734	netdev_err(priv->dev, "%s: DMA engine initialization failed\n",
	2735	__func__);
	2736	return ret;
	2737	}
	2738
	2739	/* Copy the MAC addr into the HW */
	2740	stmmac_set_umac_addr(priv, priv->hw, dev->dev_addr, 0);
	2741
	2742	/* PS and related bits will be programmed according to the speed */
	2743	if (priv->hw->pcs) {
	2744	int speed = priv->plat->mac_port_sel_speed;
	2745
	2746	if ((speed == SPEED_10) \|\| (speed == SPEED_100) \|\|
	2747	(speed == SPEED_1000)) {
	2748	priv->hw->ps = speed;
	2749	} else {
	2750	dev_warn(priv->device, "invalid port speed\n");
	2751	priv->hw->ps = 0;
	2752	}
	2753	}
	2754
	2755	/* Initialize the MAC Core */
	2756	stmmac_core_init(priv, priv->hw, dev);
	2757
	2758	/* Initialize MTL*/
	2759	stmmac_mtl_configuration(priv);
	2760
	2761	/* Initialize Safety Features */
	2762	stmmac_safety_feat_configuration(priv);
	2763
	2764	ret = stmmac_rx_ipc(priv, priv->hw);
	2765	if (!ret) {
	2766	netdev_warn(priv->dev, "RX IPC Checksum Offload disabled\n");
	2767	priv->plat->rx_coe = STMMAC_RX_COE_NONE;
	2768	priv->hw->rx_csum = 0;
	2769	}
	2770
	2771	/* Enable the MAC Rx/Tx */
	2772	stmmac_mac_set(priv, priv->ioaddr, true);
	2773
	2774	/* Set the HW DMA mode and the COE */
	2775	stmmac_dma_operation_mode(priv);
	2776
	2777	stmmac_mmc_setup(priv);
	2778
	2779	if (ptp_register) {
	2780	ret = clk_prepare_enable(priv->plat->clk_ptp_ref);
	2781	if (ret < 0)
	2782	netdev_warn(priv->dev,
	2783	"failed to enable PTP reference clock: %pe\n",
	2784	ERR_PTR(ret));
	2785	}
	2786
	2787	ret = stmmac_init_ptp(priv);
	2788	if (ret == -EOPNOTSUPP)
	2789	netdev_warn(priv->dev, "PTP not supported by HW\n");
	2790	else if (ret)
	2791	netdev_warn(priv->dev, "PTP init failed\n");
	2792	else if (ptp_register)
	2793	stmmac_ptp_register(priv);
	2794
	2795	priv->eee_tw_timer = STMMAC_DEFAULT_TWT_LS;
	2796
	2797	/* Convert the timer from msec to usec */
	2798	if (!priv->tx_lpi_timer)
	2799	priv->tx_lpi_timer = eee_timer * 1000;
	2800
	2801	if (priv->use_riwt) {
	2802	if (!priv->rx_riwt)
	2803	priv->rx_riwt = DEF_DMA_RIWT;
	2804
	2805	ret = stmmac_rx_watchdog(priv, priv->ioaddr, priv->rx_riwt, rx_cnt);
	2806	}
	2807
	2808	if (priv->hw->pcs)
	2809	stmmac_pcs_ctrl_ane(priv, priv->ioaddr, 1, priv->hw->ps, 0);
	2810
	2811	/* set TX and RX rings length */
	2812	stmmac_set_rings_length(priv);
	2813
	2814	/* Enable TSO */
	2815	if (priv->tso) {
	2816	for (chan = 0; chan < tx_cnt; chan++) {
	2817	struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan];
	2818
	2819	/* TSO and TBS cannot co-exist */
	2820	if (tx_q->tbs & STMMAC_TBS_AVAIL)
	2821	continue;
	2822
	2823	stmmac_enable_tso(priv, priv->ioaddr, 1, chan);
	2824	}
	2825	}
	2826
	2827	/* Enable Split Header */
	2828	if (priv->sph && priv->hw->rx_csum) {
	2829	for (chan = 0; chan < rx_cnt; chan++)
	2830	stmmac_enable_sph(priv, priv->ioaddr, 1, chan);
	2831	}
	2832
	2833	/* VLAN Tag Insertion */
	2834	if (priv->dma_cap.vlins)
	2835	stmmac_enable_vlan(priv, priv->hw, STMMAC_VLAN_INSERT);
	2836
	2837	/* TBS */
	2838	for (chan = 0; chan < tx_cnt; chan++) {
	2839	struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan];
	2840	int enable = tx_q->tbs & STMMAC_TBS_AVAIL;
	2841
	2842	stmmac_enable_tbs(priv, priv->ioaddr, enable, chan);
	2843	}
	2844
	2845	/* Configure real RX and TX queues */
	2846	netif_set_real_num_rx_queues(dev, priv->plat->rx_queues_to_use);
	2847	netif_set_real_num_tx_queues(dev, priv->plat->tx_queues_to_use);
	2848
	2849	/* Start the ball rolling... */
	2850	stmmac_start_all_dma(priv);
	2851
	2852	return 0;
	2853	}
	2854
	2855	static void stmmac_hw_teardown(struct net_device *dev)
	2856	{
	2857	struct stmmac_priv *priv = netdev_priv(dev);
	2858
	2859	clk_disable_unprepare(priv->plat->clk_ptp_ref);
	2860	}
	2861
	2862	/**
	2863	* stmmac_open - open entry point of the driver
	2864	* @dev : pointer to the device structure.
	2865	* Description:
	2866	* This function is the open entry point of the driver.
	2867	* Return value:
	2868	* 0 on success and an appropriate (-)ve integer as defined in errno.h
	2869	* file on failure.
	2870	*/
	2871	static int stmmac_open(struct net_device *dev)
	2872	{
	2873	struct stmmac_priv *priv = netdev_priv(dev);
	2874	int bfsize = 0;
	2875	u32 chan;
	2876	int ret;
	2877
	2878	ret = pm_runtime_get_sync(priv->device);
	2879	if (ret < 0) {
	2880	pm_runtime_put_noidle(priv->device);
	2881	return ret;
	2882	}
	2883
	2884	if (priv->hw->pcs != STMMAC_PCS_TBI &&
	2885	priv->hw->pcs != STMMAC_PCS_RTBI &&
	2886	priv->hw->xpcs == NULL) {
	2887	ret = stmmac_init_phy(dev);
	2888	if (ret) {
	2889	netdev_err(priv->dev,
	2890	"%s: Cannot attach to PHY (error: %d)\n",
	2891	__func__, ret);
	2892	goto init_phy_error;
	2893	}
	2894	}
	2895
	2896	/* Extra statistics */
	2897	memset(&priv->xstats, 0, sizeof(struct stmmac_extra_stats));
	2898	priv->xstats.threshold = tc;
	2899
	2900	bfsize = stmmac_set_16kib_bfsize(priv, dev->mtu);
	2901	if (bfsize < 0)
	2902	bfsize = 0;
	2903
	2904	if (bfsize < BUF_SIZE_16KiB)
	2905	bfsize = stmmac_set_bfsize(dev->mtu, priv->dma_buf_sz);
	2906
	2907	priv->dma_buf_sz = bfsize;
	2908	buf_sz = bfsize;
	2909
	2910	priv->rx_copybreak = STMMAC_RX_COPYBREAK;
	2911
	2912	if (!priv->dma_tx_size)
	2913	priv->dma_tx_size = priv->plat->dma_tx_size ? priv->plat->dma_tx_size :
	2914	DMA_DEFAULT_TX_SIZE;
	2915
	2916	if (!priv->dma_rx_size)
	2917	priv->dma_rx_size = priv->plat->dma_rx_size ? priv->plat->dma_rx_size :
	2918	DMA_DEFAULT_RX_SIZE;
	2919
	2920	/* Earlier check for TBS */
	2921	for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) {
	2922	struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan];
	2923	int tbs_en = priv->plat->tx_queues_cfg[chan].tbs_en;
	2924
	2925	/* Setup per-TXQ tbs flag before TX descriptor alloc */
	2926	tx_q->tbs \|= tbs_en ? STMMAC_TBS_AVAIL : 0;
	2927	}
	2928
	2929	ret = alloc_dma_desc_resources(priv);
	2930	if (ret < 0) {
	2931	netdev_err(priv->dev, "%s: DMA descriptors allocation failed\n",
	2932	__func__);
	2933	goto dma_desc_error;
	2934	}
	2935
	2936	ret = init_dma_desc_rings(dev, GFP_KERNEL);
	2937	if (ret < 0) {
	2938	netdev_err(priv->dev, "%s: DMA descriptors initialization failed\n",
	2939	__func__);
	2940	goto init_error;
	2941	}
	2942
	2943	if (priv->plat->serdes_powerup) {
	2944	ret = priv->plat->serdes_powerup(dev, priv->plat->bsp_priv);
	2945	if (ret < 0) {
	2946	netdev_err(priv->dev, "%s: Serdes powerup failed\n",
	2947	__func__);
	2948	goto init_error;
	2949	}
	2950	}
	2951
	2952	ret = stmmac_hw_setup(dev, true);
	2953	if (ret < 0) {
	2954	netdev_err(priv->dev, "%s: Hw setup failed\n", __func__);
	2955	goto init_error;
	2956	}
	2957
	2958	stmmac_init_coalesce(priv);
	2959
	2960	phylink_start(priv->phylink);
	2961	/* We may have called phylink_speed_down before */
	2962	phylink_speed_up(priv->phylink);
	2963
	2964	/* Request the IRQ lines */
	2965	ret = request_irq(dev->irq, stmmac_interrupt,
	2966	IRQF_SHARED, dev->name, dev);
	2967	if (unlikely(ret < 0)) {
	2968	netdev_err(priv->dev,
	2969	"%s: ERROR: allocating the IRQ %d (error: %d)\n",
	2970	__func__, dev->irq, ret);
	2971	goto irq_error;
	2972	}
	2973
	2974	/* Request the Wake IRQ in case of another line is used for WoL */
	2975	if (priv->wol_irq != dev->irq) {
	2976	ret = request_irq(priv->wol_irq, stmmac_interrupt,
	2977	IRQF_SHARED, dev->name, dev);
	2978	if (unlikely(ret < 0)) {
	2979	netdev_err(priv->dev,
	2980	"%s: ERROR: allocating the WoL IRQ %d (%d)\n",
	2981	__func__, priv->wol_irq, ret);
	2982	goto wolirq_error;
	2983	}
	2984	}
	2985
	2986	/* Request the IRQ lines */
	2987	if (priv->lpi_irq > 0) {
	2988	ret = request_irq(priv->lpi_irq, stmmac_interrupt, IRQF_SHARED,
	2989	dev->name, dev);
	2990	if (unlikely(ret < 0)) {
	2991	netdev_err(priv->dev,
	2992	"%s: ERROR: allocating the LPI IRQ %d (%d)\n",
	2993	__func__, priv->lpi_irq, ret);
	2994	goto lpiirq_error;
	2995	}
	2996	}
	2997
	2998	stmmac_enable_all_queues(priv);
	2999	netif_tx_start_all_queues(priv->dev);
	3000
	3001	return 0;
	3002
	3003	lpiirq_error:
	3004	if (priv->wol_irq != dev->irq)
	3005	free_irq(priv->wol_irq, dev);
	3006	wolirq_error:
	3007	free_irq(dev->irq, dev);
	3008	irq_error:
	3009	phylink_stop(priv->phylink);
	3010
	3011	for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)
	3012	del_timer_sync(&priv->tx_queue[chan].txtimer);
	3013
	3014	stmmac_hw_teardown(dev);
	3015	init_error:
	3016	free_dma_desc_resources(priv);
	3017	dma_desc_error:
	3018	phylink_disconnect_phy(priv->phylink);
	3019	init_phy_error:
	3020	pm_runtime_put(priv->device);
	3021	return ret;
	3022	}
	3023
	3024	/**
	3025	* stmmac_release - close entry point of the driver
	3026	* @dev : device pointer.
	3027	* Description:
	3028	* This is the stop entry point of the driver.
	3029	*/
	3030	static int stmmac_release(struct net_device *dev)
	3031	{
	3032	struct stmmac_priv *priv = netdev_priv(dev);
	3033	u32 chan;
	3034
	3035	if (device_may_wakeup(priv->device))
	3036	phylink_speed_down(priv->phylink, false);
	3037	/* Stop and disconnect the PHY */
	3038	phylink_stop(priv->phylink);
	3039	phylink_disconnect_phy(priv->phylink);
	3040
	3041	if (priv->plat->integrated_phy_power)
	3042	priv->plat->integrated_phy_power(priv->plat->bsp_priv, false);
	3043
	3044	stmmac_disable_all_queues(priv);
	3045
	3046	for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)
	3047	del_timer_sync(&priv->tx_queue[chan].txtimer);
	3048
	3049	/* Free the IRQ lines */
	3050	free_irq(dev->irq, dev);
	3051	if (priv->wol_irq != dev->irq)
	3052	free_irq(priv->wol_irq, dev);
	3053	if (priv->lpi_irq > 0)
	3054	free_irq(priv->lpi_irq, dev);
	3055
	3056	if (priv->eee_enabled) {
	3057	priv->tx_path_in_lpi_mode = false;
	3058	del_timer_sync(&priv->eee_ctrl_timer);
	3059	}
	3060
	3061	/* Stop TX/RX DMA and clear the descriptors */
	3062	stmmac_stop_all_dma(priv);
	3063
	3064	/* Release and free the Rx/Tx resources */
	3065	free_dma_desc_resources(priv);
	3066
	3067	/* Disable the MAC Rx/Tx */
	3068	stmmac_mac_set(priv, priv->ioaddr, false);
	3069
	3070	/* Powerdown Serdes if there is */
	3071	if (priv->plat->serdes_powerdown)
	3072	priv->plat->serdes_powerdown(dev, priv->plat->bsp_priv);
	3073
	3074	netif_carrier_off(dev);
	3075
	3076	stmmac_release_ptp(priv);
	3077
	3078	pm_runtime_put(priv->device);
	3079
	3080	return 0;
	3081	}
	3082
	3083	static bool stmmac_vlan_insert(struct stmmac_priv priv, struct sk_buff skb,
	3084	struct stmmac_tx_queue *tx_q)
	3085	{
	3086	u16 tag = 0x0, inner_tag = 0x0;
	3087	u32 inner_type = 0x0;
	3088	struct dma_desc *p;
	3089
	3090	if (!priv->dma_cap.vlins)
	3091	return false;
	3092	if (!skb_vlan_tag_present(skb))
	3093	return false;
	3094	if (skb->vlan_proto == htons(ETH_P_8021AD)) {
	3095	inner_tag = skb_vlan_tag_get(skb);
	3096	inner_type = STMMAC_VLAN_INSERT;
	3097	}
	3098
	3099	tag = skb_vlan_tag_get(skb);
	3100
	3101	if (tx_q->tbs & STMMAC_TBS_AVAIL)
	3102	p = &tx_q->dma_entx[tx_q->cur_tx].basic;
	3103	else
	3104	p = &tx_q->dma_tx[tx_q->cur_tx];
	3105
	3106	if (stmmac_set_desc_vlan_tag(priv, p, tag, inner_tag, inner_type))
	3107	return false;
	3108
	3109	stmmac_set_tx_owner(priv, p);
	3110	tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_tx_size);
	3111	return true;
	3112	}
	3113
	3114	/**
	3115	* stmmac_tso_allocator - close entry point of the driver
	3116	* @priv: driver private structure
	3117	* @des: buffer start address
	3118	* @total_len: total length to fill in descriptors
	3119	* @last_segment: condition for the last descriptor
	3120	* @queue: TX queue index
	3121	* Description:
	3122	* This function fills descriptor and request new descriptors according to
	3123	* buffer length to fill
	3124	*/
	3125	static void stmmac_tso_allocator(struct stmmac_priv *priv, dma_addr_t des,
	3126	int total_len, bool last_segment, u32 queue)
	3127	{
	3128	struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
	3129	struct dma_desc *desc;
	3130	u32 buff_size;
	3131	int tmp_len;
	3132
	3133	tmp_len = total_len;
	3134
	3135	while (tmp_len > 0) {
	3136	dma_addr_t curr_addr;
	3137
	3138	tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx,
	3139	priv->dma_tx_size);
	3140	WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]);
	3141
	3142	if (tx_q->tbs & STMMAC_TBS_AVAIL)
	3143	desc = &tx_q->dma_entx[tx_q->cur_tx].basic;
	3144	else
	3145	desc = &tx_q->dma_tx[tx_q->cur_tx];
	3146
	3147	curr_addr = des + (total_len - tmp_len);
	3148	if (priv->dma_cap.addr64 <= 32)
	3149	desc->des0 = cpu_to_le32(curr_addr);
	3150	else
	3151	stmmac_set_desc_addr(priv, desc, curr_addr);
	3152
	3153	buff_size = tmp_len >= TSO_MAX_BUFF_SIZE ?
	3154	TSO_MAX_BUFF_SIZE : tmp_len;
	3155
	3156	stmmac_prepare_tso_tx_desc(priv, desc, 0, buff_size,
	3157	0, 1,
	3158	(last_segment) && (tmp_len <= TSO_MAX_BUFF_SIZE),
	3159	0, 0);
	3160
	3161	tmp_len -= TSO_MAX_BUFF_SIZE;
	3162	}
	3163	}
	3164
	3165	/**
	3166	* stmmac_tso_xmit - Tx entry point of the driver for oversized frames (TSO)
	3167	* @skb : the socket buffer
	3168	* @dev : device pointer
	3169	* Description: this is the transmit function that is called on TSO frames
	3170	* (support available on GMAC4 and newer chips).
	3171	* Diagram below show the ring programming in case of TSO frames:
	3172	*
	3173	* First Descriptor
	3174	* --------
	3175	* \| DES0 \|---> buffer1 = L2/L3/L4 header
	3176	* \| DES1 \|---> TCP Payload (can continue on next descr...)
	3177	* \| DES2 \|---> buffer 1 and 2 len
	3178	* \| DES3 \|---> must set TSE, TCP hdr len-> [22:19]. TCP payload len [17:0]
	3179	* --------
	3180	* \|
	3181	* ...
	3182	* \|
	3183	* --------
	3184	* \| DES0 \| --\| Split TCP Payload on Buffers 1 and 2
	3185	* \| DES1 \| --\|
	3186	* \| DES2 \| --> buffer 1 and 2 len
	3187	* \| DES3 \|
	3188	* --------
	3189	*
	3190	* mss is fixed when enable tso, so w/o programming the TDES3 ctx field.
	3191	*/
	3192	static netdev_tx_t stmmac_tso_xmit(struct sk_buff skb, struct net_device dev)
	3193	{
	3194	struct dma_desc desc, first, *mss_desc = NULL;
	3195	struct stmmac_priv *priv = netdev_priv(dev);
	3196	int desc_size, tmp_pay_len = 0, first_tx;
	3197	int nfrags = skb_shinfo(skb)->nr_frags;
	3198	u32 queue = skb_get_queue_mapping(skb);
	3199	unsigned int first_entry, tx_packets;
	3200	struct stmmac_tx_queue *tx_q;
	3201	bool has_vlan, set_ic;
	3202	u8 proto_hdr_len, hdr;
	3203	u32 pay_len, mss;
	3204	dma_addr_t des;
	3205	int i;
	3206
	3207	tx_q = &priv->tx_queue[queue];
	3208	first_tx = tx_q->cur_tx;
	3209
	3210	/* Compute header lengths */
	3211	if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) {
	3212	proto_hdr_len = skb_transport_offset(skb) + sizeof(struct udphdr);
	3213	hdr = sizeof(struct udphdr);
	3214	} else {
	3215	proto_hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
	3216	hdr = tcp_hdrlen(skb);
	3217	}
	3218
	3219	/* Desc availability based on threshold should be enough safe */
	3220	if (unlikely(stmmac_tx_avail(priv, queue) <
	3221	(((skb->len - proto_hdr_len) / TSO_MAX_BUFF_SIZE + 1)))) {
	3222	if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) {
	3223	netif_tx_stop_queue(netdev_get_tx_queue(priv->dev,
	3224	queue));
	3225	/* This is a hard error, log it. */
	3226	netdev_err(priv->dev,
	3227	"%s: Tx Ring full when queue awake\n",
	3228	__func__);
	3229	}
	3230	return NETDEV_TX_BUSY;
	3231	}
	3232
	3233	pay_len = skb_headlen(skb) - proto_hdr_len; /* no frags */
	3234
	3235	mss = skb_shinfo(skb)->gso_size;
	3236
	3237	/* set new MSS value if needed */
	3238	if (mss != tx_q->mss) {
	3239	if (tx_q->tbs & STMMAC_TBS_AVAIL)
	3240	mss_desc = &tx_q->dma_entx[tx_q->cur_tx].basic;
	3241	else
	3242	mss_desc = &tx_q->dma_tx[tx_q->cur_tx];
	3243
	3244	stmmac_set_mss(priv, mss_desc, mss);
	3245	tx_q->mss = mss;
	3246	tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx,
	3247	priv->dma_tx_size);
	3248	WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]);
	3249	}
	3250
	3251	if (netif_msg_tx_queued(priv)) {
	3252	pr_info("%s: hdrlen %d, hdr_len %d, pay_len %d, mss %d\n",
	3253	__func__, hdr, proto_hdr_len, pay_len, mss);
	3254	pr_info("\tskb->len %d, skb->data_len %d\n", skb->len,
	3255	skb->data_len);
	3256	}
	3257
	3258	/* Check if VLAN can be inserted by HW */
	3259	has_vlan = stmmac_vlan_insert(priv, skb, tx_q);
	3260
	3261	first_entry = tx_q->cur_tx;
	3262	WARN_ON(tx_q->tx_skbuff[first_entry]);
	3263
	3264	if (tx_q->tbs & STMMAC_TBS_AVAIL)
	3265	desc = &tx_q->dma_entx[first_entry].basic;
	3266	else
	3267	desc = &tx_q->dma_tx[first_entry];
	3268	first = desc;
	3269
	3270	if (has_vlan)
	3271	stmmac_set_desc_vlan(priv, first, STMMAC_VLAN_INSERT);
	3272
	3273	/* first descriptor: fill Headers on Buf1 */
	3274	des = dma_map_single(priv->device, skb->data, skb_headlen(skb),
	3275	DMA_TO_DEVICE);
	3276	if (dma_mapping_error(priv->device, des))
	3277	goto dma_map_err;
	3278
	3279	tx_q->tx_skbuff_dma[first_entry].buf = des;
	3280	tx_q->tx_skbuff_dma[first_entry].len = skb_headlen(skb);
	3281
	3282	if (priv->dma_cap.addr64 <= 32) {
	3283	first->des0 = cpu_to_le32(des);
	3284
	3285	/* Fill start of payload in buff2 of first descriptor */
	3286	if (pay_len)
	3287	first->des1 = cpu_to_le32(des + proto_hdr_len);
	3288
	3289	/* If needed take extra descriptors to fill the remaining payload */
	3290	tmp_pay_len = pay_len - TSO_MAX_BUFF_SIZE;
	3291	} else {
	3292	stmmac_set_desc_addr(priv, first, des);
	3293	tmp_pay_len = pay_len;
	3294	des += proto_hdr_len;
	3295	pay_len = 0;
	3296	}
	3297
	3298	stmmac_tso_allocator(priv, des, tmp_pay_len, (nfrags == 0), queue);
	3299
	3300	/* Prepare fragments */
	3301	for (i = 0; i < nfrags; i++) {
	3302	const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
	3303
	3304	des = skb_frag_dma_map(priv->device, frag, 0,
	3305	skb_frag_size(frag),
	3306	DMA_TO_DEVICE);
	3307	if (dma_mapping_error(priv->device, des))
	3308	goto dma_map_err;
	3309
	3310	stmmac_tso_allocator(priv, des, skb_frag_size(frag),
	3311	(i == nfrags - 1), queue);
	3312
	3313	tx_q->tx_skbuff_dma[tx_q->cur_tx].buf = des;
	3314	tx_q->tx_skbuff_dma[tx_q->cur_tx].len = skb_frag_size(frag);
	3315	tx_q->tx_skbuff_dma[tx_q->cur_tx].map_as_page = true;
	3316	}
	3317
	3318	tx_q->tx_skbuff_dma[tx_q->cur_tx].last_segment = true;
	3319
	3320	/* Only the last descriptor gets to point to the skb. */
	3321	tx_q->tx_skbuff[tx_q->cur_tx] = skb;
	3322
	3323	/* Manage tx mitigation */
	3324	tx_packets = (tx_q->cur_tx + 1) - first_tx;
	3325	tx_q->tx_count_frames += tx_packets;
	3326
	3327	if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en)
	3328	set_ic = true;
	3329	else if (!priv->tx_coal_frames)
	3330	set_ic = false;
	3331	else if (tx_packets > priv->tx_coal_frames)
	3332	set_ic = true;
	3333	else if ((tx_q->tx_count_frames % priv->tx_coal_frames) < tx_packets)
	3334	set_ic = true;
	3335	else
	3336	set_ic = false;
	3337
	3338	if (set_ic) {
	3339	if (tx_q->tbs & STMMAC_TBS_AVAIL)
	3340	desc = &tx_q->dma_entx[tx_q->cur_tx].basic;
	3341	else
	3342	desc = &tx_q->dma_tx[tx_q->cur_tx];
	3343
	3344	tx_q->tx_count_frames = 0;
	3345	stmmac_set_tx_ic(priv, desc);
	3346	priv->xstats.tx_set_ic_bit++;
	3347	}
	3348
	3349	/* We've used all descriptors we need for this skb, however,
	3350	* advance cur_tx so that it references a fresh descriptor.
	3351	* ndo_start_xmit will fill this descriptor the next time it's
	3352	* called and stmmac_tx_clean may clean up to this descriptor.
	3353	*/
	3354	tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_tx_size);
	3355
	3356	if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) {
	3357	netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n",
	3358	__func__);
	3359	netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue));
	3360	}
	3361
	3362	dev->stats.tx_bytes += skb->len;
	3363	priv->xstats.tx_tso_frames++;
	3364	priv->xstats.tx_tso_nfrags += nfrags;
	3365
	3366	if (priv->sarc_type)
	3367	stmmac_set_desc_sarc(priv, first, priv->sarc_type);
	3368
	3369	skb_tx_timestamp(skb);
	3370
	3371	if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
	3372	priv->hwts_tx_en)) {
	3373	/* declare that device is doing timestamping */
	3374	skb_shinfo(skb)->tx_flags \|= SKBTX_IN_PROGRESS;
	3375	stmmac_enable_tx_timestamp(priv, first);
	3376	}
	3377
	3378	/* Complete the first descriptor before granting the DMA */
	3379	stmmac_prepare_tso_tx_desc(priv, first, 1,
	3380	proto_hdr_len,
	3381	pay_len,
	3382	1, tx_q->tx_skbuff_dma[first_entry].last_segment,
	3383	hdr / 4, (skb->len - proto_hdr_len));
	3384
	3385	/* If context desc is used to change MSS */
	3386	if (mss_desc) {
	3387	/* Make sure that first descriptor has been completely
	3388	* written, including its own bit. This is because MSS is
	3389	* actually before first descriptor, so we need to make
	3390	* sure that MSS's own bit is the last thing written.
	3391	*/
	3392	dma_wmb();
	3393	stmmac_set_tx_owner(priv, mss_desc);
	3394	}
	3395
	3396	/* The own bit must be the latest setting done when prepare the
	3397	* descriptor and then barrier is needed to make sure that
	3398	* all is coherent before granting the DMA engine.
	3399	*/
	3400	wmb();
	3401
	3402	if (netif_msg_pktdata(priv)) {
	3403	pr_info("%s: curr=%d dirty=%d f=%d, e=%d, f_p=%p, nfrags %d\n",
	3404	__func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry,
	3405	tx_q->cur_tx, first, nfrags);
	3406	pr_info(">>> frame to be transmitted: ");
	3407	print_pkt(skb->data, skb_headlen(skb));
	3408	}
	3409
	3410	netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len);
	3411
	3412	if (tx_q->tbs & STMMAC_TBS_AVAIL)
	3413	desc_size = sizeof(struct dma_edesc);
	3414	else
	3415	desc_size = sizeof(struct dma_desc);
	3416
	3417	tx_q->tx_tail_addr = tx_q->dma_tx_phy + (tx_q->cur_tx * desc_size);
	3418	stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr, queue);
	3419	stmmac_tx_timer_arm(priv, queue);
	3420
	3421	return NETDEV_TX_OK;
	3422
	3423	dma_map_err:
	3424	dev_err(priv->device, "Tx dma map failed\n");
	3425	dev_kfree_skb(skb);
	3426	priv->dev->stats.tx_dropped++;
	3427	return NETDEV_TX_OK;
	3428	}
	3429
	3430	/**
	3431	* stmmac_xmit - Tx entry point of the driver
	3432	* @skb : the socket buffer
	3433	* @dev : device pointer
	3434	* Description : this is the tx entry point of the driver.
	3435	* It programs the chain or the ring and supports oversized frames
	3436	* and SG feature.
	3437	*/
	3438	static netdev_tx_t stmmac_xmit(struct sk_buff skb, struct net_device dev)
	3439	{
	3440	unsigned int first_entry, tx_packets, enh_desc;
	3441	struct stmmac_priv *priv = netdev_priv(dev);
	3442	unsigned int nopaged_len = skb_headlen(skb);
	3443	int i, csum_insertion = 0, is_jumbo = 0;
	3444	u32 queue = skb_get_queue_mapping(skb);
	3445	int nfrags = skb_shinfo(skb)->nr_frags;
	3446	int gso = skb_shinfo(skb)->gso_type;
	3447	struct dma_edesc *tbs_desc = NULL;
	3448	int entry, desc_size, first_tx;
	3449	struct dma_desc desc, first;
	3450	struct stmmac_tx_queue *tx_q;
	3451	bool has_vlan, set_ic;
	3452	dma_addr_t des;
	3453
	3454	tx_q = &priv->tx_queue[queue];
	3455	first_tx = tx_q->cur_tx;
	3456
	3457	if (priv->tx_path_in_lpi_mode)
	3458	stmmac_disable_eee_mode(priv);
	3459
	3460	/* Manage oversized TCP frames for GMAC4 device */
	3461	if (skb_is_gso(skb) && priv->tso) {
	3462	if (gso & (SKB_GSO_TCPV4 \| SKB_GSO_TCPV6))
	3463	return stmmac_tso_xmit(skb, dev);
	3464	if (priv->plat->has_gmac4 && (gso & SKB_GSO_UDP_L4))
	3465	return stmmac_tso_xmit(skb, dev);
	3466	}
	3467
	3468	if (unlikely(stmmac_tx_avail(priv, queue) < nfrags + 1)) {
	3469	if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) {
	3470	netif_tx_stop_queue(netdev_get_tx_queue(priv->dev,
	3471	queue));
	3472	/* This is a hard error, log it. */
	3473	netdev_err(priv->dev,
	3474	"%s: Tx Ring full when queue awake\n",
	3475	__func__);
	3476	}
	3477	return NETDEV_TX_BUSY;
	3478	}
	3479
	3480	/* Check if VLAN can be inserted by HW */
	3481	has_vlan = stmmac_vlan_insert(priv, skb, tx_q);
	3482
	3483	entry = tx_q->cur_tx;
	3484	first_entry = entry;
	3485	WARN_ON(tx_q->tx_skbuff[first_entry]);
	3486
	3487	csum_insertion = (skb->ip_summed == CHECKSUM_PARTIAL);
	3488
	3489	if (likely(priv->extend_desc))
	3490	desc = (struct dma_desc *)(tx_q->dma_etx + entry);
	3491	else if (tx_q->tbs & STMMAC_TBS_AVAIL)
	3492	desc = &tx_q->dma_entx[entry].basic;
	3493	else
	3494	desc = tx_q->dma_tx + entry;
	3495
	3496	first = desc;
	3497
	3498	if (has_vlan)
	3499	stmmac_set_desc_vlan(priv, first, STMMAC_VLAN_INSERT);
	3500
	3501	enh_desc = priv->plat->enh_desc;
	3502	/* To program the descriptors according to the size of the frame */
	3503	if (enh_desc)
	3504	is_jumbo = stmmac_is_jumbo_frm(priv, skb->len, enh_desc);
	3505
	3506	if (unlikely(is_jumbo)) {
	3507	entry = stmmac_jumbo_frm(priv, tx_q, skb, csum_insertion);
	3508	if (unlikely(entry < 0) && (entry != -EINVAL))
	3509	goto dma_map_err;
	3510	}
	3511
	3512	for (i = 0; i < nfrags; i++) {
	3513	const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
	3514	int len = skb_frag_size(frag);
	3515	bool last_segment = (i == (nfrags - 1));
	3516
	3517	entry = STMMAC_GET_ENTRY(entry, priv->dma_tx_size);
	3518	WARN_ON(tx_q->tx_skbuff[entry]);
	3519
	3520	if (likely(priv->extend_desc))
	3521	desc = (struct dma_desc *)(tx_q->dma_etx + entry);
	3522	else if (tx_q->tbs & STMMAC_TBS_AVAIL)
	3523	desc = &tx_q->dma_entx[entry].basic;
	3524	else
	3525	desc = tx_q->dma_tx + entry;
	3526
	3527	des = skb_frag_dma_map(priv->device, frag, 0, len,
	3528	DMA_TO_DEVICE);
	3529	if (dma_mapping_error(priv->device, des))
	3530	goto dma_map_err; /* should reuse desc w/o issues */
	3531
	3532	tx_q->tx_skbuff_dma[entry].buf = des;
	3533
	3534	stmmac_set_desc_addr(priv, desc, des);
	3535
	3536	tx_q->tx_skbuff_dma[entry].map_as_page = true;
	3537	tx_q->tx_skbuff_dma[entry].len = len;
	3538	tx_q->tx_skbuff_dma[entry].last_segment = last_segment;
	3539
	3540	/* Prepare the descriptor and set the own bit too */
	3541	stmmac_prepare_tx_desc(priv, desc, 0, len, csum_insertion,
	3542	priv->mode, 1, last_segment, skb->len);
	3543	}
	3544
	3545	/* Only the last descriptor gets to point to the skb. */
	3546	tx_q->tx_skbuff[entry] = skb;
	3547
	3548	/* According to the coalesce parameter the IC bit for the latest
	3549	* segment is reset and the timer re-started to clean the tx status.
	3550	* This approach takes care about the fragments: desc is the first
	3551	* element in case of no SG.
	3552	*/
	3553	tx_packets = (entry + 1) - first_tx;
	3554	tx_q->tx_count_frames += tx_packets;
	3555
	3556	if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en)
	3557	set_ic = true;
	3558	else if (!priv->tx_coal_frames)
	3559	set_ic = false;
	3560	else if (tx_packets > priv->tx_coal_frames)
	3561	set_ic = true;
	3562	else if ((tx_q->tx_count_frames % priv->tx_coal_frames) < tx_packets)
	3563	set_ic = true;
	3564	else
	3565	set_ic = false;
	3566
	3567	if (set_ic) {
	3568	if (likely(priv->extend_desc))
	3569	desc = &tx_q->dma_etx[entry].basic;
	3570	else if (tx_q->tbs & STMMAC_TBS_AVAIL)
	3571	desc = &tx_q->dma_entx[entry].basic;
	3572	else
	3573	desc = &tx_q->dma_tx[entry];
	3574
	3575	tx_q->tx_count_frames = 0;
	3576	stmmac_set_tx_ic(priv, desc);
	3577	priv->xstats.tx_set_ic_bit++;
	3578	}
	3579
	3580	/* We've used all descriptors we need for this skb, however,
	3581	* advance cur_tx so that it references a fresh descriptor.
	3582	* ndo_start_xmit will fill this descriptor the next time it's
	3583	* called and stmmac_tx_clean may clean up to this descriptor.
	3584	*/
	3585	entry = STMMAC_GET_ENTRY(entry, priv->dma_tx_size);
	3586	tx_q->cur_tx = entry;
	3587
	3588	if (netif_msg_pktdata(priv)) {
	3589	netdev_dbg(priv->dev,
	3590	"%s: curr=%d dirty=%d f=%d, e=%d, first=%p, nfrags=%d",
	3591	__func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry,
	3592	entry, first, nfrags);
	3593
	3594	netdev_dbg(priv->dev, ">>> frame to be transmitted: ");
	3595	print_pkt(skb->data, skb->len);
	3596	}
	3597
	3598	if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) {
	3599	netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n",
	3600	__func__);
	3601	netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue));
	3602	}
	3603
	3604	dev->stats.tx_bytes += skb->len;
	3605
	3606	if (priv->sarc_type)
	3607	stmmac_set_desc_sarc(priv, first, priv->sarc_type);
	3608
	3609	skb_tx_timestamp(skb);
	3610
	3611	/* Ready to fill the first descriptor and set the OWN bit w/o any
	3612	* problems because all the descriptors are actually ready to be
	3613	* passed to the DMA engine.
	3614	*/
	3615	if (likely(!is_jumbo)) {
	3616	bool last_segment = (nfrags == 0);
	3617
	3618	des = dma_map_single(priv->device, skb->data,
	3619	nopaged_len, DMA_TO_DEVICE);
	3620	if (dma_mapping_error(priv->device, des))
	3621	goto dma_map_err;
	3622
	3623	tx_q->tx_skbuff_dma[first_entry].buf = des;
	3624
	3625	stmmac_set_desc_addr(priv, first, des);
	3626
	3627	tx_q->tx_skbuff_dma[first_entry].len = nopaged_len;
	3628	tx_q->tx_skbuff_dma[first_entry].last_segment = last_segment;
	3629
	3630	if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
	3631	priv->hwts_tx_en)) {
	3632	/* declare that device is doing timestamping */
	3633	skb_shinfo(skb)->tx_flags \|= SKBTX_IN_PROGRESS;
	3634	stmmac_enable_tx_timestamp(priv, first);
	3635	}
	3636
	3637	/* Prepare the first descriptor setting the OWN bit too */
	3638	stmmac_prepare_tx_desc(priv, first, 1, nopaged_len,
	3639	csum_insertion, priv->mode, 0, last_segment,
	3640	skb->len);
	3641	}
	3642
	3643	if (tx_q->tbs & STMMAC_TBS_EN) {
	3644	struct timespec64 ts = ns_to_timespec64(skb->tstamp);
	3645
	3646	tbs_desc = &tx_q->dma_entx[first_entry];
	3647	stmmac_set_desc_tbs(priv, tbs_desc, ts.tv_sec, ts.tv_nsec);
	3648	}
	3649
	3650	stmmac_set_tx_owner(priv, first);
	3651
	3652	/* The own bit must be the latest setting done when prepare the
	3653	* descriptor and then barrier is needed to make sure that
	3654	* all is coherent before granting the DMA engine.
	3655	*/
	3656	wmb();
	3657
	3658	netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len);
	3659
	3660	stmmac_enable_dma_transmission(priv, priv->ioaddr);
	3661
	3662	if (likely(priv->extend_desc))
	3663	desc_size = sizeof(struct dma_extended_desc);
	3664	else if (tx_q->tbs & STMMAC_TBS_AVAIL)
	3665	desc_size = sizeof(struct dma_edesc);
	3666	else
	3667	desc_size = sizeof(struct dma_desc);
	3668
	3669	tx_q->tx_tail_addr = tx_q->dma_tx_phy + (tx_q->cur_tx * desc_size);
	3670	stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr, queue);
	3671	stmmac_tx_timer_arm(priv, queue);
	3672
	3673	return NETDEV_TX_OK;
	3674
	3675	dma_map_err:
	3676	netdev_err(priv->dev, "Tx DMA map failed\n");
	3677	dev_kfree_skb(skb);
	3678	priv->dev->stats.tx_dropped++;
	3679	return NETDEV_TX_OK;
	3680	}
	3681
	3682	static void stmmac_rx_vlan(struct net_device dev, struct sk_buff skb)
	3683	{
	3684	struct vlan_ethhdr *veth;
	3685	__be16 vlan_proto;
	3686	u16 vlanid;
	3687
	3688	veth = (struct vlan_ethhdr *)skb->data;
	3689	vlan_proto = veth->h_vlan_proto;
	3690
	3691	if ((vlan_proto == htons(ETH_P_8021Q) &&
	3692	dev->features & NETIF_F_HW_VLAN_CTAG_RX) \|\|
	3693	(vlan_proto == htons(ETH_P_8021AD) &&
	3694	dev->features & NETIF_F_HW_VLAN_STAG_RX)) {
	3695	/* pop the vlan tag */
	3696	vlanid = ntohs(veth->h_vlan_TCI);
	3697	memmove(skb->data + VLAN_HLEN, veth, ETH_ALEN * 2);
	3698	skb_pull(skb, VLAN_HLEN);
	3699	__vlan_hwaccel_put_tag(skb, vlan_proto, vlanid);
	3700	}
	3701	}
	3702
	3703	/**
	3704	* stmmac_rx_refill - refill used skb preallocated buffers
	3705	* @priv: driver private structure
	3706	* @queue: RX queue index
	3707	* Description : this is to reallocate the skb for the reception process
	3708	* that is based on zero-copy.
	3709	*/
	3710	static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue)
	3711	{
	3712	struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
	3713	int len, dirty = stmmac_rx_dirty(priv, queue);
	3714	unsigned int entry = rx_q->dirty_rx;
	3715	gfp_t gfp = (GFP_ATOMIC \| __GFP_NOWARN);
	3716
	3717	if (priv->dma_cap.addr64 <= 32)
	3718	gfp \|= GFP_DMA32;
	3719
	3720	len = DIV_ROUND_UP(priv->dma_buf_sz, PAGE_SIZE) * PAGE_SIZE;
	3721
	3722	while (dirty-- > 0) {
	3723	struct stmmac_rx_buffer *buf = &rx_q->buf_pool[entry];
	3724	struct dma_desc *p;
	3725	bool use_rx_wd;
	3726
	3727	if (priv->extend_desc)
	3728	p = (struct dma_desc *)(rx_q->dma_erx + entry);
	3729	else
	3730	p = rx_q->dma_rx + entry;
	3731
	3732	if (!buf->page) {
	3733	buf->page = page_pool_alloc_pages(rx_q->page_pool, gfp);
	3734	if (!buf->page)
	3735	break;
	3736	}
	3737
	3738	if (priv->sph && !buf->sec_page) {
	3739	buf->sec_page = page_pool_alloc_pages(rx_q->page_pool, gfp);
	3740	if (!buf->sec_page)
	3741	break;
	3742
	3743	buf->sec_addr = page_pool_get_dma_addr(buf->sec_page);
	3744	}
	3745
	3746	buf->addr = page_pool_get_dma_addr(buf->page);
	3747	stmmac_set_desc_addr(priv, p, buf->addr);
	3748	if (priv->sph)
	3749	stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true);
	3750	else
	3751	stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, false);
	3752	stmmac_refill_desc3(priv, rx_q, p);
	3753
	3754	rx_q->rx_count_frames++;
	3755	rx_q->rx_count_frames += priv->rx_coal_frames;
	3756	if (rx_q->rx_count_frames > priv->rx_coal_frames)
	3757	rx_q->rx_count_frames = 0;
	3758
	3759	use_rx_wd = !priv->rx_coal_frames;
	3760	use_rx_wd \|= rx_q->rx_count_frames > 0;
	3761	if (!priv->use_riwt)
	3762	use_rx_wd = false;
	3763
	3764	dma_wmb();
	3765	stmmac_set_rx_owner(priv, p, use_rx_wd);
	3766
	3767	entry = STMMAC_GET_ENTRY(entry, priv->dma_rx_size);
	3768	}
	3769	rx_q->dirty_rx = entry;
	3770	rx_q->rx_tail_addr = rx_q->dma_rx_phy +
	3771	(rx_q->dirty_rx * sizeof(struct dma_desc));
	3772	stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_q->rx_tail_addr, queue);
	3773	}
	3774
	3775	static unsigned int stmmac_rx_buf1_len(struct stmmac_priv *priv,
	3776	struct dma_desc *p,
	3777	int status, unsigned int len)
	3778	{
	3779	unsigned int plen = 0, hlen = 0;
	3780	int coe = priv->hw->rx_csum;
	3781
	3782	/* Not first descriptor, buffer is always zero */
	3783	if (priv->sph && len)
	3784	return 0;
	3785
	3786	/* First descriptor, get split header length */
	3787	stmmac_get_rx_header_len(priv, p, &hlen);
	3788	if (priv->sph && hlen) {
	3789	priv->xstats.rx_split_hdr_pkt_n++;
	3790	return hlen;
	3791	}
	3792
	3793	/* First descriptor, not last descriptor and not split header */
	3794	if (status & rx_not_ls)
	3795	return priv->dma_buf_sz;
	3796
	3797	plen = stmmac_get_rx_frame_len(priv, p, coe);
	3798
	3799	/* First descriptor and last descriptor and not split header */
	3800	return min_t(unsigned int, priv->dma_buf_sz, plen);
	3801	}
	3802
	3803	static unsigned int stmmac_rx_buf2_len(struct stmmac_priv *priv,
	3804	struct dma_desc *p,
	3805	int status, unsigned int len)
	3806	{
	3807	int coe = priv->hw->rx_csum;
	3808	unsigned int plen = 0;
	3809
	3810	/* Not split header, buffer is not available */
	3811	if (!priv->sph)
	3812	return 0;
	3813
	3814	/* Not last descriptor */
	3815	if (status & rx_not_ls)
	3816	return priv->dma_buf_sz;
	3817
	3818	plen = stmmac_get_rx_frame_len(priv, p, coe);
	3819
	3820	/* Last descriptor */
	3821	return plen - len;
	3822	}
	3823
	3824	/**
	3825	* stmmac_rx - manage the receive process
	3826	* @priv: driver private structure
	3827	* @limit: napi bugget
	3828	* @queue: RX queue index.
	3829	* Description : this the function called by the napi poll method.
	3830	* It gets all the frames inside the ring.
	3831	*/
	3832	static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue)
	3833	{
	3834	struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
	3835	struct stmmac_channel *ch = &priv->channel[queue];
	3836	unsigned int count = 0, error = 0, len = 0;
	3837	int status = 0, coe = priv->hw->rx_csum;
	3838	unsigned int next_entry = rx_q->cur_rx;
	3839	unsigned int desc_size;
	3840	struct sk_buff *skb = NULL;
	3841
	3842	if (netif_msg_rx_status(priv)) {
	3843	void *rx_head;
	3844
	3845	netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__);
	3846	if (priv->extend_desc) {
	3847	rx_head = (void *)rx_q->dma_erx;
	3848	desc_size = sizeof(struct dma_extended_desc);
	3849	} else {
	3850	rx_head = (void *)rx_q->dma_rx;
	3851	desc_size = sizeof(struct dma_desc);
	3852	}
	3853
	3854	stmmac_display_ring(priv, rx_head, priv->dma_rx_size, true,
	3855	rx_q->dma_rx_phy, desc_size);
	3856	}
	3857	while (count < limit) {
	3858	unsigned int buf1_len = 0, buf2_len = 0;
	3859	enum pkt_hash_types hash_type;
	3860	struct stmmac_rx_buffer *buf;
	3861	struct dma_desc np, p;
	3862	int entry;
	3863	u32 hash;
	3864
	3865	if (!count && rx_q->state_saved) {
	3866	skb = rx_q->state.skb;
	3867	error = rx_q->state.error;
	3868	len = rx_q->state.len;
	3869	} else {
	3870	rx_q->state_saved = false;
	3871	skb = NULL;
	3872	error = 0;
	3873	len = 0;
	3874	}
	3875
	3876	if ((count >= limit - 1) && limit > 1)
	3877	break;
	3878
	3879	read_again:
	3880	buf1_len = 0;
	3881	buf2_len = 0;
	3882	entry = next_entry;
	3883	buf = &rx_q->buf_pool[entry];
	3884
	3885	if (priv->extend_desc)
	3886	p = (struct dma_desc *)(rx_q->dma_erx + entry);
	3887	else
	3888	p = rx_q->dma_rx + entry;
	3889
	3890	/* read the status of the incoming frame */
	3891	status = stmmac_rx_status(priv, &priv->dev->stats,
	3892	&priv->xstats, p);
	3893	/* check if managed by the DMA otherwise go ahead */
	3894	if (unlikely(status & dma_own))
	3895	break;
	3896
	3897	rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx,
	3898	priv->dma_rx_size);
	3899	next_entry = rx_q->cur_rx;
	3900
	3901	if (priv->extend_desc)
	3902	np = (struct dma_desc *)(rx_q->dma_erx + next_entry);
	3903	else
	3904	np = rx_q->dma_rx + next_entry;
	3905
	3906	prefetch(np);
	3907
	3908	if (priv->extend_desc)
	3909	stmmac_rx_extended_status(priv, &priv->dev->stats,
	3910	&priv->xstats, rx_q->dma_erx + entry);
	3911	if (unlikely(status == discard_frame)) {
	3912	page_pool_recycle_direct(rx_q->page_pool, buf->page);
	3913	buf->page = NULL;
	3914	error = 1;
	3915	if (!priv->hwts_rx_en)
	3916	priv->dev->stats.rx_errors++;
	3917	}
	3918
	3919	if (unlikely(error && (status & rx_not_ls)))
	3920	goto read_again;
	3921	if (unlikely(error)) {
	3922	dev_kfree_skb(skb);
	3923	skb = NULL;
	3924	count++;
	3925	continue;
	3926	}
	3927
	3928	/* Buffer is good. Go on. */
	3929
	3930	prefetch(page_address(buf->page));
	3931	if (buf->sec_page)
	3932	prefetch(page_address(buf->sec_page));
	3933
	3934	buf1_len = stmmac_rx_buf1_len(priv, p, status, len);
	3935	len += buf1_len;
	3936	buf2_len = stmmac_rx_buf2_len(priv, p, status, len);
	3937	len += buf2_len;
	3938
	3939	/* ACS is set; GMAC core strips PAD/FCS for IEEE 802.3
	3940	* Type frames (LLC/LLC-SNAP)
	3941	*
	3942	* llc_snap is never checked in GMAC >= 4, so this ACS
	3943	* feature is always disabled and packets need to be
	3944	* stripped manually.
	3945	*/
	3946	if (likely(!(status & rx_not_ls)) &&
	3947	(likely(priv->synopsys_id >= DWMAC_CORE_4_00) \|\|
	3948	unlikely(status != llc_snap))) {
	3949	if (buf2_len)
	3950	buf2_len -= ETH_FCS_LEN;
	3951	else
	3952	buf1_len -= ETH_FCS_LEN;
	3953
	3954	len -= ETH_FCS_LEN;
	3955	}
	3956
	3957	if (!skb) {
	3958	skb = napi_alloc_skb(&ch->rx_napi, buf1_len);
	3959	if (!skb) {
	3960	priv->dev->stats.rx_dropped++;
	3961	count++;
	3962	goto drain_data;
	3963	}
	3964
	3965	dma_sync_single_for_cpu(priv->device, buf->addr,
	3966	buf1_len, DMA_FROM_DEVICE);
	3967	skb_copy_to_linear_data(skb, page_address(buf->page),
	3968	buf1_len);
	3969	skb_put(skb, buf1_len);
	3970
	3971	/* Data payload copied into SKB, page ready for recycle */
	3972	page_pool_recycle_direct(rx_q->page_pool, buf->page);
	3973	buf->page = NULL;
	3974	} else if (buf1_len) {
	3975	dma_sync_single_for_cpu(priv->device, buf->addr,
	3976	buf1_len, DMA_FROM_DEVICE);
	3977	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
	3978	buf->page, 0, buf1_len,
	3979	priv->dma_buf_sz);
	3980
	3981	/* Data payload appended into SKB */
	3982	page_pool_release_page(rx_q->page_pool, buf->page);
	3983	buf->page = NULL;
	3984	}
	3985
	3986	if (buf2_len) {
	3987	dma_sync_single_for_cpu(priv->device, buf->sec_addr,
	3988	buf2_len, DMA_FROM_DEVICE);
	3989	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
	3990	buf->sec_page, 0, buf2_len,
	3991	priv->dma_buf_sz);
	3992
	3993	/* Data payload appended into SKB */
	3994	page_pool_release_page(rx_q->page_pool, buf->sec_page);
	3995	buf->sec_page = NULL;
	3996	}
	3997
	3998	drain_data:
	3999	if (likely(status & rx_not_ls))
	4000	goto read_again;
	4001	if (!skb)
	4002	continue;
	4003
	4004	/* Got entire packet into SKB. Finish it. */
	4005
	4006	stmmac_get_rx_hwtstamp(priv, p, np, skb);
	4007	stmmac_rx_vlan(priv->dev, skb);
	4008	skb->protocol = eth_type_trans(skb, priv->dev);
	4009
	4010	if (unlikely(!coe))
	4011	skb_checksum_none_assert(skb);
	4012	else
	4013	skb->ip_summed = CHECKSUM_UNNECESSARY;
	4014
	4015	if (!stmmac_get_rx_hash(priv, p, &hash, &hash_type))
	4016	skb_set_hash(skb, hash, hash_type);
	4017
	4018	skb_record_rx_queue(skb, queue);
	4019	napi_gro_receive(&ch->rx_napi, skb);
	4020	skb = NULL;
	4021
	4022	priv->dev->stats.rx_packets++;
	4023	priv->dev->stats.rx_bytes += len;
	4024	count++;
	4025	}
	4026
	4027	if (status & rx_not_ls \|\| skb) {
	4028	rx_q->state_saved = true;
	4029	rx_q->state.skb = skb;
	4030	rx_q->state.error = error;
	4031	rx_q->state.len = len;
	4032	}
	4033
	4034	stmmac_rx_refill(priv, queue);
	4035
	4036	priv->xstats.rx_pkt_n += count;
	4037
	4038	return count;
	4039	}
	4040
	4041	static int stmmac_napi_poll_rx(struct napi_struct *napi, int budget)
	4042	{
	4043	struct stmmac_channel *ch =
	4044	container_of(napi, struct stmmac_channel, rx_napi);
	4045	struct stmmac_priv *priv = ch->priv_data;
	4046	u32 chan = ch->index;
	4047	int work_done;
	4048
	4049	priv->xstats.napi_poll++;
	4050
	4051	work_done = stmmac_rx(priv, budget, chan);
	4052	if (work_done < budget && napi_complete_done(napi, work_done)) {
	4053	unsigned long flags;
	4054
	4055	spin_lock_irqsave(&ch->lock, flags);
	4056	stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 0);
	4057	spin_unlock_irqrestore(&ch->lock, flags);
	4058	}
	4059
	4060	return work_done;
	4061	}
	4062
	4063	static int stmmac_napi_poll_tx(struct napi_struct *napi, int budget)
	4064	{
	4065	struct stmmac_channel *ch =
	4066	container_of(napi, struct stmmac_channel, tx_napi);
	4067	struct stmmac_priv *priv = ch->priv_data;
	4068	u32 chan = ch->index;
	4069	int work_done;
	4070
	4071	priv->xstats.napi_poll++;
	4072
	4073	work_done = stmmac_tx_clean(priv, priv->dma_tx_size, chan);
	4074	work_done = min(work_done, budget);
	4075
	4076	if (work_done < budget && napi_complete_done(napi, work_done)) {
	4077	unsigned long flags;
	4078
	4079	spin_lock_irqsave(&ch->lock, flags);
	4080	stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 0, 1);
	4081	spin_unlock_irqrestore(&ch->lock, flags);
	4082	}
	4083
	4084	return work_done;
	4085	}
	4086
	4087	/**
	4088	* stmmac_tx_timeout
	4089	* @dev : Pointer to net device structure
	4090	* @txqueue: the index of the hanging transmit queue
	4091	* Description: this function is called when a packet transmission fails to
	4092	* complete within a reasonable time. The driver will mark the error in the
	4093	* netdev structure and arrange for the device to be reset to a sane state
	4094	* in order to transmit a new packet.
	4095	*/
	4096	static void stmmac_tx_timeout(struct net_device *dev, unsigned int txqueue)
	4097	{
	4098	struct stmmac_priv *priv = netdev_priv(dev);
	4099
	4100	stmmac_global_err(priv);
	4101	}
	4102
	4103	/**
	4104	* stmmac_set_rx_mode - entry point for multicast addressing
	4105	* @dev : pointer to the device structure
	4106	* Description:
	4107	* This function is a driver entry point which gets called by the kernel
	4108	* whenever multicast addresses must be enabled/disabled.
	4109	* Return value:
	4110	* void.
	4111	*/
	4112	static void stmmac_set_rx_mode(struct net_device *dev)
	4113	{
	4114	struct stmmac_priv *priv = netdev_priv(dev);
	4115
	4116	stmmac_set_filter(priv, priv->hw, dev);
	4117	}
	4118
	4119	/**
	4120	* stmmac_change_mtu - entry point to change MTU size for the device.
	4121	* @dev : device pointer.
	4122	* @new_mtu : the new MTU size for the device.
	4123	* Description: the Maximum Transfer Unit (MTU) is used by the network layer
	4124	* to drive packet transmission. Ethernet has an MTU of 1500 octets
	4125	* (ETH_DATA_LEN). This value can be changed with ifconfig.
	4126	* Return value:
	4127	* 0 on success and an appropriate (-)ve integer as defined in errno.h
	4128	* file on failure.
	4129	*/
	4130	static int stmmac_change_mtu(struct net_device *dev, int new_mtu)
	4131	{
	4132	struct stmmac_priv *priv = netdev_priv(dev);
	4133	int txfifosz = priv->plat->tx_fifo_size;
	4134	const int mtu = new_mtu;
	4135
	4136	if (txfifosz == 0)
	4137	txfifosz = priv->dma_cap.tx_fifo_size;
	4138
	4139	txfifosz /= priv->plat->tx_queues_to_use;
	4140
	4141	if (netif_running(dev)) {
	4142	netdev_err(priv->dev, "must be stopped to change its MTU\n");
	4143	return -EBUSY;
	4144	}
	4145
	4146	new_mtu = STMMAC_ALIGN(new_mtu);
	4147
	4148	/* If condition true, FIFO is too small or MTU too large */
	4149	if ((txfifosz < new_mtu) \|\| (new_mtu > BUF_SIZE_16KiB))
	4150	return -EINVAL;
	4151
	4152	dev->mtu = mtu;
	4153
	4154	netdev_update_features(dev);
	4155
	4156	return 0;
	4157	}
	4158
	4159	static netdev_features_t stmmac_fix_features(struct net_device *dev,
	4160	netdev_features_t features)
	4161	{
	4162	struct stmmac_priv *priv = netdev_priv(dev);
	4163
	4164	if (priv->plat->rx_coe == STMMAC_RX_COE_NONE)
	4165	features &= ~NETIF_F_RXCSUM;
	4166
	4167	if (!priv->plat->tx_coe)
	4168	features &= ~NETIF_F_CSUM_MASK;
	4169
	4170	/* Some GMAC devices have a bugged Jumbo frame support that
	4171	* needs to have the Tx COE disabled for oversized frames
	4172	* (due to limited buffer sizes). In this case we disable
	4173	* the TX csum insertion in the TDES and not use SF.
	4174	*/
	4175	if (priv->plat->bugged_jumbo && (dev->mtu > ETH_DATA_LEN))
	4176	features &= ~NETIF_F_CSUM_MASK;
	4177
	4178	/* Disable tso if asked by ethtool */
	4179	if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) {
	4180	if (features & NETIF_F_TSO)
	4181	priv->tso = true;
	4182	else
	4183	priv->tso = false;
	4184	}
	4185
	4186	return features;
	4187	}
	4188
	4189	static int stmmac_set_features(struct net_device *netdev,
	4190	netdev_features_t features)
	4191	{
	4192	struct stmmac_priv *priv = netdev_priv(netdev);
	4193	bool sph_en;
	4194	u32 chan;
	4195
	4196	/* Keep the COE Type in case of csum is supporting */
	4197	if (features & NETIF_F_RXCSUM)
	4198	priv->hw->rx_csum = priv->plat->rx_coe;
	4199	else
	4200	priv->hw->rx_csum = 0;
	4201	/* No check needed because rx_coe has been set before and it will be
	4202	* fixed in case of issue.
	4203	*/
	4204	stmmac_rx_ipc(priv, priv->hw);
	4205
	4206	sph_en = (priv->hw->rx_csum > 0) && priv->sph;
	4207	for (chan = 0; chan < priv->plat->rx_queues_to_use; chan++)
	4208	stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan);
	4209
	4210	return 0;
	4211	}
	4212
	4213	/**
	4214	* stmmac_interrupt - main ISR
	4215	* @irq: interrupt number.
	4216	* @dev_id: to pass the net device pointer (must be valid).
	4217	* Description: this is the main driver interrupt service routine.
	4218	* It can call:
	4219	* o DMA service routine (to manage incoming frame reception and transmission
	4220	* status)
	4221	* o Core interrupts to manage: remote wake-up, management counter, LPI
	4222	* interrupts.
	4223	*/
	4224	static irqreturn_t stmmac_interrupt(int irq, void *dev_id)
	4225	{
	4226	struct net_device dev = (struct net_device )dev_id;
	4227	struct stmmac_priv *priv = netdev_priv(dev);
	4228	u32 rx_cnt = priv->plat->rx_queues_to_use;
	4229	u32 tx_cnt = priv->plat->tx_queues_to_use;
	4230	u32 queues_count;
	4231	u32 queue;
	4232	bool xmac;
	4233
	4234	xmac = priv->plat->has_gmac4 \|\| priv->plat->has_xgmac;
	4235	queues_count = (rx_cnt > tx_cnt) ? rx_cnt : tx_cnt;
	4236
	4237	if (priv->irq_wake)
	4238	pm_wakeup_event(priv->device, 0);
	4239
	4240	/* Check if adapter is up */
	4241	if (test_bit(STMMAC_DOWN, &priv->state))
	4242	return IRQ_HANDLED;
	4243	/* Check if a fatal error happened */
	4244	if (stmmac_safety_feat_interrupt(priv))
	4245	return IRQ_HANDLED;
	4246
	4247	/* To handle GMAC own interrupts */
	4248	if ((priv->plat->has_gmac) \|\| xmac) {
	4249	int status = stmmac_host_irq_status(priv, priv->hw, &priv->xstats);
	4250
	4251	if (unlikely(status)) {
	4252	/* For LPI we need to save the tx status */
	4253	if (status & CORE_IRQ_TX_PATH_IN_LPI_MODE)
	4254	priv->tx_path_in_lpi_mode = true;
	4255	if (status & CORE_IRQ_TX_PATH_EXIT_LPI_MODE)
	4256	priv->tx_path_in_lpi_mode = false;
	4257	}
	4258
	4259	for (queue = 0; queue < queues_count; queue++) {
	4260	status = stmmac_host_mtl_irq_status(priv, priv->hw,
	4261	queue);
	4262	}
	4263
	4264	/* PCS link status */
	4265	if (priv->hw->pcs) {
	4266	if (priv->xstats.pcs_link)
	4267	netif_carrier_on(dev);
	4268	else
	4269	netif_carrier_off(dev);
	4270	}
	4271	}
	4272
	4273	/* To handle DMA interrupts */
	4274	stmmac_dma_interrupt(priv);
	4275
	4276	return IRQ_HANDLED;
	4277	}
	4278
	4279	#ifdef CONFIG_NET_POLL_CONTROLLER
	4280	/* Polling receive - used by NETCONSOLE and other diagnostic tools
	4281	* to allow network I/O with interrupts disabled.
	4282	*/
	4283	static void stmmac_poll_controller(struct net_device *dev)
	4284	{
	4285	disable_irq(dev->irq);
	4286	stmmac_interrupt(dev->irq, dev);
	4287	enable_irq(dev->irq);
	4288	}
	4289	#endif
	4290
	4291	/**
	4292	* stmmac_ioctl - Entry point for the Ioctl
	4293	* @dev: Device pointer.
	4294	* @rq: An IOCTL specefic structure, that can contain a pointer to
	4295	* a proprietary structure used to pass information to the driver.
	4296	* @cmd: IOCTL command
	4297	* Description:
	4298	* Currently it supports the phy_mii_ioctl(...) and HW time stamping.
	4299	*/
	4300	static int stmmac_ioctl(struct net_device dev, struct ifreq rq, int cmd)
	4301	{
	4302	struct stmmac_priv *priv = netdev_priv (dev);
	4303	int ret = -EOPNOTSUPP;
	4304
	4305	if (!netif_running(dev))
	4306	return -EINVAL;
	4307
	4308	switch (cmd) {
	4309	case SIOCGMIIPHY:
	4310	case SIOCGMIIREG:
	4311	case SIOCSMIIREG:
	4312	ret = phylink_mii_ioctl(priv->phylink, rq, cmd);
	4313	break;
	4314	case SIOCSHWTSTAMP:
	4315	ret = stmmac_hwtstamp_set(dev, rq);
	4316	break;
	4317	case SIOCGHWTSTAMP:
	4318	ret = stmmac_hwtstamp_get(dev, rq);
	4319	break;
	4320	default:
	4321	break;
	4322	}
	4323
	4324	return ret;
	4325	}
	4326
	4327	static int stmmac_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
	4328	void *cb_priv)
	4329	{
	4330	struct stmmac_priv *priv = cb_priv;
	4331	int ret = -EOPNOTSUPP;
	4332
	4333	if (!tc_cls_can_offload_and_chain0(priv->dev, type_data))
	4334	return ret;
	4335
	4336	stmmac_disable_all_queues(priv);
	4337
	4338	switch (type) {
	4339	case TC_SETUP_CLSU32:
	4340	ret = stmmac_tc_setup_cls_u32(priv, priv, type_data);
	4341	break;
	4342	case TC_SETUP_CLSFLOWER:
	4343	ret = stmmac_tc_setup_cls(priv, priv, type_data);
	4344	break;
	4345	default:
	4346	break;
	4347	}
	4348
	4349	stmmac_enable_all_queues(priv);
	4350	return ret;
	4351	}
	4352
	4353	static LIST_HEAD(stmmac_block_cb_list);
	4354
	4355	static int stmmac_setup_tc(struct net_device *ndev, enum tc_setup_type type,
	4356	void *type_data)
	4357	{
	4358	struct stmmac_priv *priv = netdev_priv(ndev);
	4359
	4360	switch (type) {
	4361	case TC_SETUP_BLOCK:
	4362	return flow_block_cb_setup_simple(type_data,
	4363	&stmmac_block_cb_list,
	4364	stmmac_setup_tc_block_cb,
	4365	priv, priv, true);
	4366	case TC_SETUP_QDISC_CBS:
	4367	return stmmac_tc_setup_cbs(priv, priv, type_data);
	4368	case TC_SETUP_QDISC_TAPRIO:
	4369	return stmmac_tc_setup_taprio(priv, priv, type_data);
	4370	case TC_SETUP_QDISC_ETF:
	4371	return stmmac_tc_setup_etf(priv, priv, type_data);
	4372	default:
	4373	return -EOPNOTSUPP;
	4374	}
	4375	}
	4376
	4377	static u16 stmmac_select_queue(struct net_device dev, struct sk_buff skb,
	4378	struct net_device *sb_dev)
	4379	{
	4380	int gso = skb_shinfo(skb)->gso_type;
	4381
	4382	if (gso & (SKB_GSO_TCPV4 \| SKB_GSO_TCPV6 \| SKB_GSO_UDP_L4)) {
	4383	/*
	4384	* There is no way to determine the number of TSO/USO
	4385	* capable Queues. Let's use always the Queue 0
	4386	* because if TSO/USO is supported then at least this
	4387	* one will be capable.
	4388	*/
	4389	return 0;
	4390	}
	4391
	4392	return netdev_pick_tx(dev, skb, NULL) % dev->real_num_tx_queues;
	4393	}
	4394
	4395	static int stmmac_set_mac_address(struct net_device ndev, void addr)
	4396	{
	4397	struct stmmac_priv *priv = netdev_priv(ndev);
	4398	int ret = 0;
	4399
	4400	ret = pm_runtime_get_sync(priv->device);
	4401	if (ret < 0) {
	4402	pm_runtime_put_noidle(priv->device);
	4403	return ret;
	4404	}
	4405
	4406	ret = eth_mac_addr(ndev, addr);
	4407	if (ret)
	4408	goto set_mac_error;
	4409
	4410	stmmac_set_umac_addr(priv, priv->hw, ndev->dev_addr, 0);
	4411
	4412	set_mac_error:
	4413	pm_runtime_put(priv->device);
	4414
	4415	return ret;
	4416	}
	4417
	4418	#ifdef CONFIG_DEBUG_FS
	4419	static struct dentry *stmmac_fs_dir;
	4420
	4421	static void sysfs_display_ring(void *head, int size, int extend_desc,
	4422	struct seq_file *seq, dma_addr_t dma_phy_addr)
	4423	{
	4424	int i;
	4425	struct dma_extended_desc ep = (struct dma_extended_desc )head;
	4426	struct dma_desc p = (struct dma_desc )head;
	4427	dma_addr_t dma_addr;
	4428
	4429	for (i = 0; i < size; i++) {
	4430	if (extend_desc) {
	4431	dma_addr = dma_phy_addr + i * sizeof(*ep);
	4432	seq_printf(seq, "%d [%pad]: 0x%x 0x%x 0x%x 0x%x\n",
	4433	i, &dma_addr,
	4434	le32_to_cpu(ep->basic.des0),
	4435	le32_to_cpu(ep->basic.des1),
	4436	le32_to_cpu(ep->basic.des2),
	4437	le32_to_cpu(ep->basic.des3));
	4438	ep++;
	4439	} else {
	4440	dma_addr = dma_phy_addr + i * sizeof(*p);
	4441	seq_printf(seq, "%d [%pad]: 0x%x 0x%x 0x%x 0x%x\n",
	4442	i, &dma_addr,
	4443	le32_to_cpu(p->des0), le32_to_cpu(p->des1),
	4444	le32_to_cpu(p->des2), le32_to_cpu(p->des3));
	4445	p++;
	4446	}
	4447	seq_printf(seq, "\n");
	4448	}
	4449	}
	4450
	4451	static int stmmac_rings_status_show(struct seq_file seq, void v)
	4452	{
	4453	struct net_device *dev = seq->private;
	4454	struct stmmac_priv *priv = netdev_priv(dev);
	4455	u32 rx_count = priv->plat->rx_queues_to_use;
	4456	u32 tx_count = priv->plat->tx_queues_to_use;
	4457	u32 queue;
	4458
	4459	if ((dev->flags & IFF_UP) == 0)
	4460	return 0;
	4461
	4462	for (queue = 0; queue < rx_count; queue++) {
	4463	struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
	4464
	4465	seq_printf(seq, "RX Queue %d:\n", queue);
	4466
	4467	if (priv->extend_desc) {
	4468	seq_printf(seq, "Extended descriptor ring:\n");
	4469	sysfs_display_ring((void *)rx_q->dma_erx,
	4470	priv->dma_rx_size, 1, seq, rx_q->dma_rx_phy);
	4471	} else {
	4472	seq_printf(seq, "Descriptor ring:\n");
	4473	sysfs_display_ring((void *)rx_q->dma_rx,
	4474	priv->dma_rx_size, 0, seq, rx_q->dma_rx_phy);
	4475	}
	4476	}
	4477
	4478	for (queue = 0; queue < tx_count; queue++) {
	4479	struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
	4480
	4481	seq_printf(seq, "TX Queue %d:\n", queue);
	4482
	4483	if (priv->extend_desc) {
	4484	seq_printf(seq, "Extended descriptor ring:\n");
	4485	sysfs_display_ring((void *)tx_q->dma_etx,
	4486	priv->dma_tx_size, 1, seq, tx_q->dma_tx_phy);
	4487	} else if (!(tx_q->tbs & STMMAC_TBS_AVAIL)) {
	4488	seq_printf(seq, "Descriptor ring:\n");
	4489	sysfs_display_ring((void *)tx_q->dma_tx,
	4490	priv->dma_tx_size, 0, seq, tx_q->dma_tx_phy);
	4491	}
	4492	}
	4493
	4494	return 0;
	4495	}
	4496	DEFINE_SHOW_ATTRIBUTE(stmmac_rings_status);
	4497
	4498	static int stmmac_dma_cap_show(struct seq_file seq, void v)
	4499	{
	4500	struct net_device *dev = seq->private;
	4501	struct stmmac_priv *priv = netdev_priv(dev);
	4502
	4503	if (!priv->hw_cap_support) {
	4504	seq_printf(seq, "DMA HW features not supported\n");
	4505	return 0;
	4506	}
	4507
	4508	seq_printf(seq, "==============================\n");
	4509	seq_printf(seq, "\tDMA HW features\n");
	4510	seq_printf(seq, "==============================\n");
	4511
	4512	seq_printf(seq, "\t10/100 Mbps: %s\n",
	4513	(priv->dma_cap.mbps_10_100) ? "Y" : "N");
	4514	seq_printf(seq, "\t1000 Mbps: %s\n",
	4515	(priv->dma_cap.mbps_1000) ? "Y" : "N");
	4516	seq_printf(seq, "\tHalf duplex: %s\n",
	4517	(priv->dma_cap.half_duplex) ? "Y" : "N");
	4518	seq_printf(seq, "\tHash Filter: %s\n",
	4519	(priv->dma_cap.hash_filter) ? "Y" : "N");
	4520	seq_printf(seq, "\tMultiple MAC address registers: %s\n",
	4521	(priv->dma_cap.multi_addr) ? "Y" : "N");
	4522	seq_printf(seq, "\tPCS (TBI/SGMII/RTBI PHY interfaces): %s\n",
	4523	(priv->dma_cap.pcs) ? "Y" : "N");
	4524	seq_printf(seq, "\tSMA (MDIO) Interface: %s\n",
	4525	(priv->dma_cap.sma_mdio) ? "Y" : "N");
	4526	seq_printf(seq, "\tPMT Remote wake up: %s\n",
	4527	(priv->dma_cap.pmt_remote_wake_up) ? "Y" : "N");
	4528	seq_printf(seq, "\tPMT Magic Frame: %s\n",
	4529	(priv->dma_cap.pmt_magic_frame) ? "Y" : "N");
	4530	seq_printf(seq, "\tRMON module: %s\n",
	4531	(priv->dma_cap.rmon) ? "Y" : "N");
	4532	seq_printf(seq, "\tIEEE 1588-2002 Time Stamp: %s\n",
	4533	(priv->dma_cap.time_stamp) ? "Y" : "N");
	4534	seq_printf(seq, "\tIEEE 1588-2008 Advanced Time Stamp: %s\n",
	4535	(priv->dma_cap.atime_stamp) ? "Y" : "N");
	4536	seq_printf(seq, "\t802.3az - Energy-Efficient Ethernet (EEE): %s\n",
	4537	(priv->dma_cap.eee) ? "Y" : "N");
	4538	seq_printf(seq, "\tAV features: %s\n", (priv->dma_cap.av) ? "Y" : "N");
	4539	seq_printf(seq, "\tChecksum Offload in TX: %s\n",
	4540	(priv->dma_cap.tx_coe) ? "Y" : "N");
	4541	if (priv->synopsys_id >= DWMAC_CORE_4_00) {
	4542	seq_printf(seq, "\tIP Checksum Offload in RX: %s\n",
	4543	(priv->dma_cap.rx_coe) ? "Y" : "N");
	4544	} else {
	4545	seq_printf(seq, "\tIP Checksum Offload (type1) in RX: %s\n",
	4546	(priv->dma_cap.rx_coe_type1) ? "Y" : "N");
	4547	seq_printf(seq, "\tIP Checksum Offload (type2) in RX: %s\n",
	4548	(priv->dma_cap.rx_coe_type2) ? "Y" : "N");
	4549	}
	4550	seq_printf(seq, "\tRXFIFO > 2048bytes: %s\n",
	4551	(priv->dma_cap.rxfifo_over_2048) ? "Y" : "N");
	4552	seq_printf(seq, "\tNumber of Additional RX channel: %d\n",
	4553	priv->dma_cap.number_rx_channel);
	4554	seq_printf(seq, "\tNumber of Additional TX channel: %d\n",
	4555	priv->dma_cap.number_tx_channel);
	4556	seq_printf(seq, "\tNumber of Additional RX queues: %d\n",
	4557	priv->dma_cap.number_rx_queues);
	4558	seq_printf(seq, "\tNumber of Additional TX queues: %d\n",
	4559	priv->dma_cap.number_tx_queues);
	4560	seq_printf(seq, "\tEnhanced descriptors: %s\n",
	4561	(priv->dma_cap.enh_desc) ? "Y" : "N");
	4562	seq_printf(seq, "\tTX Fifo Size: %d\n", priv->dma_cap.tx_fifo_size);
	4563	seq_printf(seq, "\tRX Fifo Size: %d\n", priv->dma_cap.rx_fifo_size);
	4564	seq_printf(seq, "\tHash Table Size: %d\n", priv->dma_cap.hash_tb_sz);
	4565	seq_printf(seq, "\tTSO: %s\n", priv->dma_cap.tsoen ? "Y" : "N");
	4566	seq_printf(seq, "\tNumber of PPS Outputs: %d\n",
	4567	priv->dma_cap.pps_out_num);
	4568	seq_printf(seq, "\tSafety Features: %s\n",
	4569	priv->dma_cap.asp ? "Y" : "N");
	4570	seq_printf(seq, "\tFlexible RX Parser: %s\n",
	4571	priv->dma_cap.frpsel ? "Y" : "N");
	4572	seq_printf(seq, "\tEnhanced Addressing: %d\n",
	4573	priv->dma_cap.addr64);
	4574	seq_printf(seq, "\tReceive Side Scaling: %s\n",
	4575	priv->dma_cap.rssen ? "Y" : "N");
	4576	seq_printf(seq, "\tVLAN Hash Filtering: %s\n",
	4577	priv->dma_cap.vlhash ? "Y" : "N");
	4578	seq_printf(seq, "\tSplit Header: %s\n",
	4579	priv->dma_cap.sphen ? "Y" : "N");
	4580	seq_printf(seq, "\tVLAN TX Insertion: %s\n",
	4581	priv->dma_cap.vlins ? "Y" : "N");
	4582	seq_printf(seq, "\tDouble VLAN: %s\n",
	4583	priv->dma_cap.dvlan ? "Y" : "N");
	4584	seq_printf(seq, "\tNumber of L3/L4 Filters: %d\n",
	4585	priv->dma_cap.l3l4fnum);
	4586	seq_printf(seq, "\tARP Offloading: %s\n",
	4587	priv->dma_cap.arpoffsel ? "Y" : "N");
	4588	seq_printf(seq, "\tEnhancements to Scheduled Traffic (EST): %s\n",
	4589	priv->dma_cap.estsel ? "Y" : "N");
	4590	seq_printf(seq, "\tFrame Preemption (FPE): %s\n",
	4591	priv->dma_cap.fpesel ? "Y" : "N");
	4592	seq_printf(seq, "\tTime-Based Scheduling (TBS): %s\n",
	4593	priv->dma_cap.tbssel ? "Y" : "N");
	4594	return 0;
	4595	}
	4596	DEFINE_SHOW_ATTRIBUTE(stmmac_dma_cap);
	4597
	4598	/* Use network device events to rename debugfs file entries.
	4599	*/
	4600	static int stmmac_device_event(struct notifier_block *unused,
	4601	unsigned long event, void *ptr)
	4602	{
	4603	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
	4604	struct stmmac_priv *priv = netdev_priv(dev);
	4605
	4606	if (dev->netdev_ops != &stmmac_netdev_ops)
	4607	goto done;
	4608
	4609	switch (event) {
	4610	case NETDEV_CHANGENAME:
	4611	if (priv->dbgfs_dir)
	4612	priv->dbgfs_dir = debugfs_rename(stmmac_fs_dir,
	4613	priv->dbgfs_dir,
	4614	stmmac_fs_dir,
	4615	dev->name);
	4616	break;
	4617	}
	4618	done:
	4619	return NOTIFY_DONE;
	4620	}
	4621
	4622	static struct notifier_block stmmac_notifier = {
	4623	.notifier_call = stmmac_device_event,
	4624	};
	4625
	4626	static void stmmac_init_fs(struct net_device *dev)
	4627	{
	4628	struct stmmac_priv *priv = netdev_priv(dev);
	4629
	4630	rtnl_lock();
	4631
	4632	/* Create per netdev entries */
	4633	priv->dbgfs_dir = debugfs_create_dir(dev->name, stmmac_fs_dir);
	4634
	4635	/* Entry to report DMA RX/TX rings */
	4636	debugfs_create_file("descriptors_status", 0444, priv->dbgfs_dir, dev,
	4637	&stmmac_rings_status_fops);
	4638
	4639	/* Entry to report the DMA HW features */
	4640	debugfs_create_file("dma_cap", 0444, priv->dbgfs_dir, dev,
	4641	&stmmac_dma_cap_fops);
	4642
	4643	rtnl_unlock();
	4644	}
	4645
	4646	static void stmmac_exit_fs(struct net_device *dev)
	4647	{
	4648	struct stmmac_priv *priv = netdev_priv(dev);
	4649
	4650	debugfs_remove_recursive(priv->dbgfs_dir);
	4651	}
	4652	#endif /* CONFIG_DEBUG_FS */
	4653
	4654	static u32 stmmac_vid_crc32_le(__le16 vid_le)
	4655	{
	4656	unsigned char data = (unsigned char )&vid_le;
	4657	unsigned char data_byte = 0;
	4658	u32 crc = ~0x0;
	4659	u32 temp = 0;
	4660	int i, bits;
	4661
	4662	bits = get_bitmask_order(VLAN_VID_MASK);
	4663	for (i = 0; i < bits; i++) {
	4664	if ((i % 8) == 0)
	4665	data_byte = data[i / 8];
	4666
	4667	temp = ((crc & 1) ^ data_byte) & 1;
	4668	crc >>= 1;
	4669	data_byte >>= 1;
	4670
	4671	if (temp)
	4672	crc ^= 0xedb88320;
	4673	}
	4674
	4675	return crc;
	4676	}
	4677
	4678	static int stmmac_vlan_update(struct stmmac_priv *priv, bool is_double)
	4679	{
	4680	u32 crc, hash = 0;
	4681	__le16 pmatch = 0;
	4682	int count = 0;
	4683	u16 vid = 0;
	4684
	4685	for_each_set_bit(vid, priv->active_vlans, VLAN_N_VID) {
	4686	__le16 vid_le = cpu_to_le16(vid);
	4687	crc = bitrev32(~stmmac_vid_crc32_le(vid_le)) >> 28;
	4688	hash \|= (1 << crc);
	4689	count++;
	4690	}
	4691
	4692	if (!priv->dma_cap.vlhash) {
	4693	if (count > 2) /* VID = 0 always passes filter */
	4694	return -EOPNOTSUPP;
	4695
	4696	pmatch = cpu_to_le16(vid);
	4697	hash = 0;
	4698	}
	4699
	4700	return stmmac_update_vlan_hash(priv, priv->hw, hash, pmatch, is_double);
	4701	}
	4702
	4703	static int stmmac_vlan_rx_add_vid(struct net_device *ndev, __be16 proto, u16 vid)
	4704	{
	4705	struct stmmac_priv *priv = netdev_priv(ndev);
	4706	bool is_double = false;
	4707	int ret;
	4708
	4709	if (be16_to_cpu(proto) == ETH_P_8021AD)
	4710	is_double = true;
	4711
	4712	set_bit(vid, priv->active_vlans);
	4713	ret = stmmac_vlan_update(priv, is_double);
	4714	if (ret) {
	4715	clear_bit(vid, priv->active_vlans);
	4716	return ret;
	4717	}
	4718
	4719	if (priv->hw->num_vlan) {
	4720	ret = stmmac_add_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid);
	4721	if (ret)
	4722	return ret;
	4723	}
	4724
	4725	return 0;
	4726	}
	4727
	4728	static int stmmac_vlan_rx_kill_vid(struct net_device *ndev, __be16 proto, u16 vid)
	4729	{
	4730	struct stmmac_priv *priv = netdev_priv(ndev);
	4731	bool is_double = false;
	4732	int ret;
	4733
	4734	ret = pm_runtime_get_sync(priv->device);
	4735	if (ret < 0) {
	4736	pm_runtime_put_noidle(priv->device);
	4737	return ret;
	4738	}
	4739
	4740	if (be16_to_cpu(proto) == ETH_P_8021AD)
	4741	is_double = true;
	4742
	4743	clear_bit(vid, priv->active_vlans);
	4744
	4745	if (priv->hw->num_vlan) {
	4746	ret = stmmac_del_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid);
	4747	if (ret)
	4748	goto del_vlan_error;
	4749	}
	4750
	4751	ret = stmmac_vlan_update(priv, is_double);
	4752
	4753	del_vlan_error:
	4754	pm_runtime_put(priv->device);
	4755
	4756	return ret;
	4757	}
	4758
	4759	static const struct net_device_ops stmmac_netdev_ops = {
	4760	.ndo_open = stmmac_open,
	4761	.ndo_start_xmit = stmmac_xmit,
	4762	.ndo_stop = stmmac_release,
	4763	.ndo_change_mtu = stmmac_change_mtu,
	4764	.ndo_fix_features = stmmac_fix_features,
	4765	.ndo_set_features = stmmac_set_features,
	4766	.ndo_set_rx_mode = stmmac_set_rx_mode,
	4767	.ndo_tx_timeout = stmmac_tx_timeout,
	4768	.ndo_do_ioctl = stmmac_ioctl,
	4769	.ndo_setup_tc = stmmac_setup_tc,
	4770	.ndo_select_queue = stmmac_select_queue,
	4771	#ifdef CONFIG_NET_POLL_CONTROLLER
	4772	.ndo_poll_controller = stmmac_poll_controller,
	4773	#endif
	4774	.ndo_set_mac_address = stmmac_set_mac_address,
	4775	.ndo_vlan_rx_add_vid = stmmac_vlan_rx_add_vid,
	4776	.ndo_vlan_rx_kill_vid = stmmac_vlan_rx_kill_vid,
	4777	};
	4778
	4779	static void stmmac_reset_subtask(struct stmmac_priv *priv)
	4780	{
	4781	if (!test_and_clear_bit(STMMAC_RESET_REQUESTED, &priv->state))
	4782	return;
	4783	if (test_bit(STMMAC_DOWN, &priv->state))
	4784	return;
	4785
	4786	netdev_err(priv->dev, "Reset adapter.\n");
	4787
	4788	rtnl_lock();
	4789	netif_trans_update(priv->dev);
	4790	while (test_and_set_bit(STMMAC_RESETING, &priv->state))
	4791	usleep_range(1000, 2000);
	4792
	4793	set_bit(STMMAC_DOWN, &priv->state);
	4794	dev_close(priv->dev);
	4795	dev_open(priv->dev, NULL);
	4796	clear_bit(STMMAC_DOWN, &priv->state);
	4797	clear_bit(STMMAC_RESETING, &priv->state);
	4798	rtnl_unlock();
	4799	}
	4800
	4801	static void stmmac_service_task(struct work_struct *work)
	4802	{
	4803	struct stmmac_priv *priv = container_of(work, struct stmmac_priv,
	4804	service_task);
	4805
	4806	stmmac_reset_subtask(priv);
	4807	clear_bit(STMMAC_SERVICE_SCHED, &priv->state);
	4808	}
	4809
	4810	/**
	4811	* stmmac_hw_init - Init the MAC device
	4812	* @priv: driver private structure
	4813	* Description: this function is to configure the MAC device according to
	4814	* some platform parameters or the HW capability register. It prepares the
	4815	* driver to use either ring or chain modes and to setup either enhanced or
	4816	* normal descriptors.
	4817	*/
	4818	static int stmmac_hw_init(struct stmmac_priv *priv)
	4819	{
	4820	int ret;
	4821
	4822	/* dwmac-sun8i only work in chain mode */
	4823	if (priv->plat->has_sun8i)
	4824	chain_mode = 1;
	4825	priv->chain_mode = chain_mode;
	4826
	4827	/* Initialize HW Interface */
	4828	ret = stmmac_hwif_init(priv);
	4829	if (ret)
	4830	return ret;
	4831
	4832	/* Get the HW capability (new GMAC newer than 3.50a) */
	4833	priv->hw_cap_support = stmmac_get_hw_features(priv);
	4834	if (priv->hw_cap_support) {
	4835	dev_info(priv->device, "DMA HW capability register supported\n");
	4836
	4837	/* We can override some gmac/dma configuration fields: e.g.
	4838	* enh_desc, tx_coe (e.g. that are passed through the
	4839	* platform) with the values from the HW capability
	4840	* register (if supported).
	4841	*/
	4842	priv->plat->enh_desc = priv->dma_cap.enh_desc;
	4843	priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up;
	4844	priv->hw->pmt = priv->plat->pmt;
	4845	if (priv->dma_cap.hash_tb_sz) {
	4846	priv->hw->multicast_filter_bins =
	4847	(BIT(priv->dma_cap.hash_tb_sz) << 5);
	4848	priv->hw->mcast_bits_log2 =
	4849	ilog2(priv->hw->multicast_filter_bins);
	4850	}
	4851
	4852	/* TXCOE doesn't work in thresh DMA mode */
	4853	if (priv->plat->force_thresh_dma_mode)
	4854	priv->plat->tx_coe = 0;
	4855	else
	4856	priv->plat->tx_coe = priv->dma_cap.tx_coe;
	4857
	4858	/* In case of GMAC4 rx_coe is from HW cap register. */
	4859	priv->plat->rx_coe = priv->dma_cap.rx_coe;
	4860
	4861	if (priv->dma_cap.rx_coe_type2)
	4862	priv->plat->rx_coe = STMMAC_RX_COE_TYPE2;
	4863	else if (priv->dma_cap.rx_coe_type1)
	4864	priv->plat->rx_coe = STMMAC_RX_COE_TYPE1;
	4865
	4866	} else {
	4867	dev_info(priv->device, "No HW DMA feature register supported\n");
	4868	}
	4869
	4870	if (priv->plat->rx_coe) {
	4871	priv->hw->rx_csum = priv->plat->rx_coe;
	4872	dev_info(priv->device, "RX Checksum Offload Engine supported\n");
	4873	if (priv->synopsys_id < DWMAC_CORE_4_00)
	4874	dev_info(priv->device, "COE Type %d\n", priv->hw->rx_csum);
	4875	}
	4876	if (priv->plat->tx_coe)
	4877	dev_info(priv->device, "TX Checksum insertion supported\n");
	4878
	4879	if (priv->plat->pmt) {
	4880	dev_info(priv->device, "Wake-Up On Lan supported\n");
	4881	device_set_wakeup_capable(priv->device, 1);
	4882	}
	4883
	4884	if (priv->dma_cap.tsoen)
	4885	dev_info(priv->device, "TSO supported\n");
	4886
	4887	priv->hw->vlan_fail_q_en = priv->plat->vlan_fail_q_en;
	4888	priv->hw->vlan_fail_q = priv->plat->vlan_fail_q;
	4889
	4890	/* Run HW quirks, if any */
	4891	if (priv->hwif_quirks) {
	4892	ret = priv->hwif_quirks(priv);
	4893	if (ret)
	4894	return ret;
	4895	}
	4896
	4897	/* Rx Watchdog is available in the COREs newer than the 3.40.
	4898	* In some case, for example on bugged HW this feature
	4899	* has to be disable and this can be done by passing the
	4900	* riwt_off field from the platform.
	4901	*/
	4902	if (((priv->synopsys_id >= DWMAC_CORE_3_50) \|\|
	4903	(priv->plat->has_xgmac)) && (!priv->plat->riwt_off)) {
	4904	priv->use_riwt = 1;
	4905	dev_info(priv->device,
	4906	"Enable RX Mitigation via HW Watchdog Timer\n");
	4907	}
	4908
	4909	return 0;
	4910	}
	4911
	4912	static void stmmac_napi_add(struct net_device *dev)
	4913	{
	4914	struct stmmac_priv *priv = netdev_priv(dev);
	4915	u32 queue, maxq;
	4916
	4917	maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use);
	4918
	4919	for (queue = 0; queue < maxq; queue++) {
	4920	struct stmmac_channel *ch = &priv->channel[queue];
	4921	int rx_budget = ((priv->plat->dma_rx_size < NAPI_POLL_WEIGHT) &&
	4922	(priv->plat->dma_rx_size > 0)) ?
	4923	priv->plat->dma_rx_size : NAPI_POLL_WEIGHT;
	4924	int tx_budget = ((priv->plat->dma_tx_size < NAPI_POLL_WEIGHT) &&
	4925	(priv->plat->dma_tx_size > 0)) ?
	4926	priv->plat->dma_tx_size : NAPI_POLL_WEIGHT;
	4927
	4928	ch->priv_data = priv;
	4929	ch->index = queue;
	4930	spin_lock_init(&ch->lock);
	4931
	4932	if (queue < priv->plat->rx_queues_to_use) {
	4933	netif_napi_add(dev, &ch->rx_napi, stmmac_napi_poll_rx,
	4934	rx_budget);
	4935	}
	4936	if (queue < priv->plat->tx_queues_to_use) {
	4937	netif_tx_napi_add(dev, &ch->tx_napi,
	4938	stmmac_napi_poll_tx, tx_budget);
	4939	}
	4940	}
	4941	}
	4942
	4943	static void stmmac_napi_del(struct net_device *dev)
	4944	{
	4945	struct stmmac_priv *priv = netdev_priv(dev);
	4946	u32 queue, maxq;
	4947
	4948	maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use);
	4949
	4950	for (queue = 0; queue < maxq; queue++) {
	4951	struct stmmac_channel *ch = &priv->channel[queue];
	4952
	4953	if (queue < priv->plat->rx_queues_to_use)
	4954	netif_napi_del(&ch->rx_napi);
	4955	if (queue < priv->plat->tx_queues_to_use)
	4956	netif_napi_del(&ch->tx_napi);
	4957	}
	4958	}
	4959
	4960	int stmmac_reinit_queues(struct net_device *dev, u32 rx_cnt, u32 tx_cnt)
	4961	{
	4962	struct stmmac_priv *priv = netdev_priv(dev);
	4963	int ret = 0;
	4964
	4965	if (netif_running(dev))
	4966	stmmac_release(dev);
	4967
	4968	stmmac_napi_del(dev);
	4969
	4970	priv->plat->rx_queues_to_use = rx_cnt;
	4971	priv->plat->tx_queues_to_use = tx_cnt;
	4972
	4973	stmmac_napi_add(dev);
	4974
	4975	if (netif_running(dev))
	4976	ret = stmmac_open(dev);
	4977
	4978	return ret;
	4979	}
	4980
	4981	int stmmac_reinit_ringparam(struct net_device *dev, u32 rx_size, u32 tx_size)
	4982	{
	4983	struct stmmac_priv *priv = netdev_priv(dev);
	4984	int ret = 0;
	4985
	4986	if (netif_running(dev))
	4987	stmmac_release(dev);
	4988
	4989	priv->dma_rx_size = rx_size;
	4990	priv->dma_tx_size = tx_size;
	4991
	4992	if (netif_running(dev))
	4993	ret = stmmac_open(dev);
	4994
	4995	return ret;
	4996	}
	4997
	4998	#ifdef LED_FIX
	4999	static int phy_dp83848_led_fixup(struct phy_device *phydev)
	5000	{
	5001	int value;
	5002
	5003	if (phydev->phy_id != DP_83848_PHY_ID)
	5004	return 0;
	5005
	5006	printk("%s in\n", __func__);
	5007
	5008	value = phy_read(phydev, 0x18);
	5009	value &= ~(1<<2);
	5010	phy_write(phydev, 0x18, value);
	5011
	5012	value = phy_read(phydev, 0x19);
	5013	value &= ~(1<<5);
	5014	phy_write(phydev, 0x19, value);
	5015
	5016	return 0;
	5017	}
	5018
	5019	static int phy_rtl8201f_led_fixup(struct phy_device *phydev)
	5020	{
	5021	int value;
	5022
	5023	printk("%s in\n", __func__);
	5024
	5025	/* switch to page 7 */
	5026	value = phy_read(phydev, 31);
	5027	value &= 0xff00;
	5028	value \|= 0x0007;
	5029	value = phy_write(phydev, 31, value);
	5030
	5031	/* set customized led enable */
	5032	value = phy_read(phydev, 19);
	5033	value \|= (0x1<<3);
	5034	phy_write(phydev, 19, value);
	5035
	5036	value &= 0x0000;
	5037	value \|= (0x1<<1);
	5038	value \|= (0x1<<7);
	5039	phy_write(phydev, 17, value);
	5040
	5041	/* back to page 0 */
	5042	value = phy_read(phydev, 31);
	5043	value &= 0x0000;
	5044	value = phy_write(phydev, 31, value);
	5045
	5046	return 0;
	5047	}
	5048
	5049	static int phy_rtl8211e_led_fixup(struct phy_device *phydev)
	5050	{
	5051	int val;
	5052
	5053	printk("%s in\n", __func__);
	5054	val = phy_read(phydev, 3);
	5055	printk("%s in val=0x%04x\n", __func__, val);
	5056
	5057	/switch to extension page44/
	5058	phy_write(phydev, 31, 0x07);
	5059	phy_write(phydev, 30, 0x2c);
	5060
	5061	/set led1(yellow) act/
	5062	val = phy_read(phydev, 26);
	5063	val &= (~(1<<4));// bit4=0
	5064	val \|= (1<<5);// bit5=1
	5065	val &= (~(1<<6));// bit6=0
	5066	phy_write(phydev, 26, val);
	5067
	5068	/set led0(green) link/
	5069	val = phy_read(phydev, 28);
	5070	val \|= (7<<0);// bit0,1,2=1
	5071	val &= (~(7<<4));// bit4,5,6=0
	5072	val &= (~(7<<8));// bit8,9,10=0
	5073	phy_write(phydev, 28, val);
	5074
	5075	/switch back to page0/
	5076	phy_write(phydev,31,0x00);
	5077
	5078	return 0;
	5079	}
	5080
	5081	static int phy_rtl8211f_led_fixup(struct phy_device *phydev)
	5082	{
	5083	int val;
	5084
	5085	printk("%s in\n", __func__);
	5086
	5087	val = phy_read(phydev, 31);
	5088	printk("%s in val=0x%04x\n", __func__, val);
	5089
	5090	/switch to extension page31/
	5091	phy_write(phydev, 31, 0xd04);
	5092
d5ef2f	5093	phy_write(phydev, 0x10, 0x0d3B); /led1-green led2-yellow /
f258bb	5094	//phy_write(phydev, 0x10, 0xc160); /led1-green led2-yellow /
d5ef2f	5095	phy_write(phydev, 0x11, 0x6002); /led0 EEE enable/
a07526	5096
H	5097	/switch back to page0/
	5098	phy_write(phydev,31,0x0);
	5099
	5100	return 0;
	5101	}
	5102	#endif
	5103
	5104	/**
	5105	* stmmac_dvr_probe
	5106	* @device: device pointer
	5107	* @plat_dat: platform data pointer
	5108	* @res: stmmac resource pointer
	5109	* Description: this is the main probe function used to
	5110	* call the alloc_etherdev, allocate the priv structure.
	5111	* Return:
	5112	* returns 0 on success, otherwise errno.
	5113	*/
	5114	int stmmac_dvr_probe(struct device *device,
	5115	struct plat_stmmacenet_data *plat_dat,
	5116	struct stmmac_resources *res)
	5117	{
	5118	struct net_device *ndev = NULL;
	5119	struct stmmac_priv *priv;
	5120	u32 rxq;
	5121	int i, ret = 0;
	5122
	5123	ndev = devm_alloc_etherdev_mqs(device, sizeof(struct stmmac_priv),
	5124	MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES);
	5125	if (!ndev)
	5126	return -ENOMEM;
	5127
	5128	SET_NETDEV_DEV(ndev, device);
	5129
	5130	priv = netdev_priv(ndev);
	5131	priv->device = device;
	5132	priv->dev = ndev;
	5133
	5134	stmmac_set_ethtool_ops(ndev);
	5135	priv->pause = pause;
	5136	priv->plat = plat_dat;
	5137	priv->ioaddr = res->addr;
	5138	priv->dev->base_addr = (unsigned long)res->addr;
	5139
	5140	priv->dev->irq = res->irq;
	5141	priv->wol_irq = res->wol_irq;
	5142	priv->lpi_irq = res->lpi_irq;
	5143
	5144	if (!IS_ERR_OR_NULL(res->mac))
	5145	memcpy(priv->dev->dev_addr, res->mac, ETH_ALEN);
	5146
	5147	dev_set_drvdata(device, priv->dev);
	5148
	5149	/* Verify driver arguments */
	5150	stmmac_verify_args();
	5151
	5152	/* Allocate workqueue */
	5153	priv->wq = create_singlethread_workqueue("stmmac_wq");
	5154	if (!priv->wq) {
	5155	dev_err(priv->device, "failed to create workqueue\n");
	5156	return -ENOMEM;
	5157	}
	5158
	5159	INIT_WORK(&priv->service_task, stmmac_service_task);
	5160
	5161	/* Override with kernel parameters if supplied XXX CRS XXX
	5162	* this needs to have multiple instances
	5163	*/
	5164	if ((phyaddr >= 0) && (phyaddr <= 31))
	5165	priv->plat->phy_addr = phyaddr;
	5166
	5167	if (priv->plat->stmmac_rst) {
	5168	ret = reset_control_assert(priv->plat->stmmac_rst);
	5169	reset_control_deassert(priv->plat->stmmac_rst);
	5170	/* Some reset controllers have only reset callback instead of
	5171	* assert + deassert callbacks pair.
	5172	*/
	5173	if (ret == -ENOTSUPP)
	5174	reset_control_reset(priv->plat->stmmac_rst);
	5175	}
	5176
	5177	/* Init MAC and get the capabilities */
	5178	ret = stmmac_hw_init(priv);
	5179	if (ret)
	5180	goto error_hw_init;
	5181
	5182	stmmac_check_ether_addr(priv);
	5183
	5184	ndev->netdev_ops = &stmmac_netdev_ops;
	5185
	5186	ndev->hw_features = NETIF_F_SG \| NETIF_F_IP_CSUM \| NETIF_F_IPV6_CSUM \|
	5187	NETIF_F_RXCSUM;
	5188
	5189	ret = stmmac_tc_init(priv, priv);
	5190	if (!ret) {
	5191	ndev->hw_features \|= NETIF_F_HW_TC;
	5192	}
	5193
	5194	if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) {
	5195	ndev->hw_features \|= NETIF_F_TSO \| NETIF_F_TSO6;
	5196	if (priv->plat->has_gmac4)
	5197	ndev->hw_features \|= NETIF_F_GSO_UDP_L4;
	5198	priv->tso = true;
	5199	dev_info(priv->device, "TSO feature enabled\n");
	5200	}
	5201
	5202	if (priv->dma_cap.sphen && !priv->plat->sph_disable) {
	5203	ndev->hw_features \|= NETIF_F_GRO;
	5204	if (!priv->plat->sph_disable) {
	5205	priv->sph = true;
	5206	dev_info(priv->device, "SPH feature enabled\n");
	5207	}
	5208	}
	5209
	5210	/* The current IP register MAC_HW_Feature1[ADDR64] only define
	5211	* 32/40/64 bit width, but some SOC support others like i.MX8MP
	5212	* support 34 bits but it map to 40 bits width in MAC_HW_Feature1[ADDR64].
	5213	* So overwrite dma_cap.addr64 according to HW real design.
	5214	*/
	5215	if (priv->plat->addr64)
	5216	priv->dma_cap.addr64 = priv->plat->addr64;
	5217
	5218	if (priv->dma_cap.addr64) {
	5219	ret = dma_set_mask_and_coherent(device,
	5220	DMA_BIT_MASK(priv->dma_cap.addr64));
	5221	if (!ret) {
	5222	dev_info(priv->device, "Using %d bits DMA width\n",
	5223	priv->dma_cap.addr64);
	5224
	5225	/*
	5226	* If more than 32 bits can be addressed, make sure to
	5227	* enable enhanced addressing mode.
	5228	*/
	5229	if (IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT))
	5230	priv->plat->dma_cfg->eame = true;
	5231	} else {
	5232	ret = dma_set_mask_and_coherent(device, DMA_BIT_MASK(32));
	5233	if (ret) {
	5234	dev_err(priv->device, "Failed to set DMA Mask\n");
	5235	goto error_hw_init;
	5236	}
	5237
	5238	priv->dma_cap.addr64 = 32;
	5239	}
	5240	}
	5241
	5242	ndev->features \|= ndev->hw_features \| NETIF_F_HIGHDMA;
	5243	ndev->watchdog_timeo = msecs_to_jiffies(watchdog);
	5244	#ifdef STMMAC_VLAN_TAG_USED
	5245	/* Both mac100 and gmac support receive VLAN tag detection */
	5246	ndev->features \|= NETIF_F_HW_VLAN_CTAG_RX \| NETIF_F_HW_VLAN_STAG_RX;
	5247	if (priv->dma_cap.vlhash) {
	5248	ndev->features \|= NETIF_F_HW_VLAN_CTAG_FILTER;
	5249	ndev->features \|= NETIF_F_HW_VLAN_STAG_FILTER;
	5250	}
	5251	if (priv->dma_cap.vlins) {
	5252	ndev->features \|= NETIF_F_HW_VLAN_CTAG_TX;
	5253	if (priv->dma_cap.dvlan)
	5254	ndev->features \|= NETIF_F_HW_VLAN_STAG_TX;
	5255	}
	5256	#endif
	5257	priv->msg_enable = netif_msg_init(debug, default_msg_level);
	5258
	5259	/* Initialize RSS */
	5260	rxq = priv->plat->rx_queues_to_use;
	5261	netdev_rss_key_fill(priv->rss.key, sizeof(priv->rss.key));
	5262	for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++)
	5263	priv->rss.table[i] = ethtool_rxfh_indir_default(i, rxq);
	5264
	5265	if (priv->dma_cap.rssen && priv->plat->rss_en)
	5266	ndev->features \|= NETIF_F_RXHASH;
	5267
	5268	/* MTU range: 46 - hw-specific max */
	5269	ndev->min_mtu = ETH_ZLEN - ETH_HLEN;
	5270	if (priv->plat->has_xgmac)
	5271	ndev->max_mtu = XGMAC_JUMBO_LEN;
	5272	else if ((priv->plat->enh_desc) \|\| (priv->synopsys_id >= DWMAC_CORE_4_00))
	5273	ndev->max_mtu = JUMBO_LEN;
	5274	else
	5275	ndev->max_mtu = SKB_MAX_HEAD(NET_SKB_PAD + NET_IP_ALIGN);
	5276	/* Will not overwrite ndev->max_mtu if plat->maxmtu > ndev->max_mtu
	5277	* as well as plat->maxmtu < ndev->min_mtu which is a invalid range.
	5278	*/
	5279	if ((priv->plat->maxmtu < ndev->max_mtu) &&
	5280	(priv->plat->maxmtu >= ndev->min_mtu))
	5281	ndev->max_mtu = priv->plat->maxmtu;
	5282	else if (priv->plat->maxmtu < ndev->min_mtu)
	5283	dev_warn(priv->device,
	5284	"%s: warning: maxmtu having invalid value (%d)\n",
	5285	__func__, priv->plat->maxmtu);
	5286
	5287	if (flow_ctrl)
	5288	priv->flow_ctrl = FLOW_AUTO; /* RX/TX pause on */
	5289
	5290	/* Setup channels NAPI */
	5291	stmmac_napi_add(ndev);
	5292
	5293	mutex_init(&priv->lock);
	5294
	5295	/* If a specific clk_csr value is passed from the platform
	5296	* this means that the CSR Clock Range selection cannot be
	5297	* changed at run-time and it is fixed. Viceversa the driver'll try to
	5298	* set the MDC clock dynamically according to the csr actual
	5299	* clock input.
	5300	*/
	5301	if (priv->plat->clk_csr >= 0)
	5302	priv->clk_csr = priv->plat->clk_csr;
	5303	else
	5304	stmmac_clk_csr_set(priv);
	5305
	5306	stmmac_check_pcs_mode(priv);
	5307
	5308	pm_runtime_get_noresume(device);
	5309	pm_runtime_set_active(device);
	5310	pm_runtime_enable(device);
	5311
	5312	if (priv->hw->pcs != STMMAC_PCS_TBI &&
	5313	priv->hw->pcs != STMMAC_PCS_RTBI) {
	5314	/* MDIO bus Registration */
	5315	ret = stmmac_mdio_register(ndev);
	5316	if (ret < 0) {
	5317	dev_err(priv->device,
	5318	"%s: MDIO bus (id: %d) registration failed",
	5319	__func__, priv->plat->bus_id);
	5320	goto error_mdio_register;
	5321	}
	5322	}
	5323
a46a1a	5324	snprintf(ndev->name, IFNAMSIZ, ndev->name, 3);
H	5325	printk(">>> troy fixup ndev->name=%s\n", ndev->name);
a07526	5326	ret = stmmac_phy_setup(priv);
H	5327	if (ret) {
	5328	netdev_err(ndev, "failed to setup phy (%d)\n", ret);
	5329	goto error_phy_setup;
	5330	}
	5331
	5332	ret = register_netdev(ndev);
	5333	if (ret) {
	5334	dev_err(priv->device, "%s: ERROR %i registering the device\n",
	5335	__func__, ret);
	5336	goto error_netdev_register;
	5337	}
	5338
	5339	#ifdef LED_FIX
	5340	/* register the PHY board fixup */
	5341	ret = phy_register_fixup_for_uid(RTL_8211E_PHY_ID, 0xffffffff, phy_rtl8211e_led_fixup);
	5342	if (ret)
	5343	pr_warn("Cannot register PHY board fixup.\n");
	5344	ret = phy_register_fixup_for_uid(RTL_8211F_PHY_ID, 0xffffffff, phy_rtl8211f_led_fixup);
	5345	if (ret)
	5346	pr_warn("Cannot register PHY board fixup.\n");
	5347	ret = phy_register_fixup_for_uid(RTL_8201F_PHY_ID, 0xffffffff, phy_rtl8201f_led_fixup);
	5348	if (ret)
	5349	pr_warn("Cannot register PHY board fixup.\n");
	5350	ret = phy_register_fixup_for_uid(DP_83848_PHY_ID, 0xffffffff, phy_dp83848_led_fixup);
	5351	if (ret)
	5352	pr_warn("Cannot register PHY board fixup.\n");
	5353	#endif
	5354
	5355	#ifdef CONFIG_DEBUG_FS
	5356	stmmac_init_fs(ndev);
	5357	#endif
	5358
	5359	/* Let pm_runtime_put() disable the clocks.
	5360	* If CONFIG_PM is not enabled, the clocks will stay powered.
	5361	*/
	5362	pm_runtime_put(device);
	5363
	5364	return ret;
	5365
	5366	error_netdev_register:
	5367	phylink_destroy(priv->phylink);
	5368	error_phy_setup:
	5369	if (priv->hw->pcs != STMMAC_PCS_TBI &&
	5370	priv->hw->pcs != STMMAC_PCS_RTBI)
	5371	stmmac_mdio_unregister(ndev);
	5372	error_mdio_register:
	5373	stmmac_napi_del(ndev);
	5374	error_hw_init:
	5375	destroy_workqueue(priv->wq);
	5376
	5377	return ret;
	5378	}
	5379	EXPORT_SYMBOL_GPL(stmmac_dvr_probe);
	5380
	5381	/**
	5382	* stmmac_dvr_remove
	5383	* @dev: device pointer
	5384	* Description: this function resets the TX/RX processes, disables the MAC RX/TX
	5385	* changes the link status, releases the DMA descriptor rings.
	5386	*/
	5387	int stmmac_dvr_remove(struct device *dev)
	5388	{
	5389	struct net_device *ndev = dev_get_drvdata(dev);
	5390	struct stmmac_priv *priv = netdev_priv(ndev);
	5391
	5392	netdev_info(priv->dev, "%s: removing driver", __func__);
	5393
	5394	stmmac_stop_all_dma(priv);
	5395	stmmac_mac_set(priv, priv->ioaddr, false);
	5396	netif_carrier_off(ndev);
	5397	unregister_netdev(ndev);
	5398
	5399	/* Serdes power down needs to happen after VLAN filter
	5400	* is deleted that is triggered by unregister_netdev().
	5401	*/
	5402	if (priv->plat->serdes_powerdown)
	5403	priv->plat->serdes_powerdown(ndev, priv->plat->bsp_priv);
	5404
	5405	#ifdef CONFIG_DEBUG_FS
	5406	stmmac_exit_fs(ndev);
	5407	#endif
	5408	phylink_destroy(priv->phylink);
	5409	if (priv->plat->stmmac_rst)
	5410	reset_control_assert(priv->plat->stmmac_rst);
	5411	pm_runtime_put(dev);
	5412	pm_runtime_disable(dev);
	5413	if (priv->hw->pcs != STMMAC_PCS_TBI &&
	5414	priv->hw->pcs != STMMAC_PCS_RTBI)
	5415	stmmac_mdio_unregister(ndev);
	5416	destroy_workqueue(priv->wq);
	5417	mutex_destroy(&priv->lock);
	5418
	5419	return 0;
	5420	}
	5421	EXPORT_SYMBOL_GPL(stmmac_dvr_remove);
	5422
	5423	/**
	5424	* stmmac_suspend - suspend callback
	5425	* @dev: device pointer
	5426	* Description: this is the function to suspend the device and it is called
	5427	* by the platform driver to stop the network queue, release the resources,
	5428	* program the PMT register (for WoL), clean and release driver resources.
	5429	*/
	5430	int stmmac_suspend(struct device *dev)
	5431	{
	5432	struct net_device *ndev = dev_get_drvdata(dev);
	5433	struct stmmac_priv *priv = netdev_priv(ndev);
	5434	u32 chan;
	5435
	5436	if (!ndev \|\| !netif_running(ndev))
	5437	return 0;
	5438
	5439	phylink_mac_change(priv->phylink, false);
	5440
	5441	mutex_lock(&priv->lock);
	5442
	5443	netif_device_detach(ndev);
	5444
	5445	stmmac_disable_all_queues(priv);
	5446
	5447	for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)
	5448	del_timer_sync(&priv->tx_queue[chan].txtimer);
	5449
	5450	if (priv->eee_enabled) {
	5451	priv->tx_path_in_lpi_mode = false;
	5452	del_timer_sync(&priv->eee_ctrl_timer);
	5453	}
	5454
	5455	/* Stop TX/RX DMA */
	5456	stmmac_stop_all_dma(priv);
	5457
	5458	if (priv->plat->serdes_powerdown)
	5459	priv->plat->serdes_powerdown(ndev, priv->plat->bsp_priv);
	5460
	5461	/* Enable Power down mode by programming the PMT regs */
	5462	if (device_may_wakeup(priv->device) && priv->plat->pmt) {
	5463	stmmac_pmt(priv, priv->hw, priv->wolopts);
	5464	priv->irq_wake = 1;
	5465	} else {
	5466	mutex_unlock(&priv->lock);
	5467	rtnl_lock();
	5468	if (device_may_wakeup(priv->device))
	5469	phylink_speed_down(priv->phylink, false);
	5470	if (priv->plat->integrated_phy_power)
	5471	priv->plat->integrated_phy_power(priv->plat->bsp_priv,
	5472	false);
	5473	phylink_stop(priv->phylink);
	5474	rtnl_unlock();
	5475	mutex_lock(&priv->lock);
	5476
	5477	stmmac_mac_set(priv, priv->ioaddr, false);
	5478	pinctrl_pm_select_sleep_state(priv->device);
	5479	}
	5480	mutex_unlock(&priv->lock);
	5481
	5482	priv->speed = SPEED_UNKNOWN;
	5483	return 0;
	5484	}
	5485	EXPORT_SYMBOL_GPL(stmmac_suspend);
	5486
	5487	/**
	5488	* stmmac_reset_queues_param - reset queue parameters
	5489	* @priv: device pointer
	5490	*/
	5491	static void stmmac_reset_queues_param(struct stmmac_priv *priv)
	5492	{
	5493	u32 rx_cnt = priv->plat->rx_queues_to_use;
	5494	u32 tx_cnt = priv->plat->tx_queues_to_use;
	5495	u32 queue;
	5496
	5497	for (queue = 0; queue < rx_cnt; queue++) {
	5498	struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
	5499
	5500	rx_q->cur_rx = 0;
	5501	rx_q->dirty_rx = 0;
	5502	}
	5503
	5504	for (queue = 0; queue < tx_cnt; queue++) {
	5505	struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
	5506
	5507	tx_q->cur_tx = 0;
	5508	tx_q->dirty_tx = 0;
	5509	tx_q->mss = 0;
	5510
	5511	netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, queue));
	5512	}
	5513	}
	5514
	5515	/**
	5516	* stmmac_resume - resume callback
	5517	* @dev: device pointer
	5518	* Description: when resume this function is invoked to setup the DMA and CORE
	5519	* in a usable state.
	5520	*/
	5521	int stmmac_resume(struct device *dev)
	5522	{
	5523	struct net_device *ndev = dev_get_drvdata(dev);
	5524	struct stmmac_priv *priv = netdev_priv(ndev);
	5525	int ret;
	5526
	5527	if (!netif_running(ndev))
	5528	return 0;
	5529
	5530	/* Power Down bit, into the PM register, is cleared
	5531	* automatically as soon as a magic packet or a Wake-up frame
	5532	* is received. Anyway, it's better to manually clear
	5533	* this bit because it can generate problems while resuming
	5534	* from another devices (e.g. serial console).
	5535	*/
	5536	if (device_may_wakeup(priv->device) && priv->plat->pmt) {
	5537	mutex_lock(&priv->lock);
	5538	stmmac_pmt(priv, priv->hw, 0);
	5539	mutex_unlock(&priv->lock);
	5540	priv->irq_wake = 0;
	5541	} else {
	5542	pinctrl_pm_select_default_state(priv->device);
	5543	/* reset the phy so that it's ready */
	5544	if (priv->mii)
	5545	stmmac_mdio_reset(priv->mii);
	5546	if (priv->plat->integrated_phy_power)
	5547	priv->plat->integrated_phy_power(priv->plat->bsp_priv,
	5548	true);
	5549	}
	5550
	5551	if (priv->plat->serdes_powerup) {
	5552	ret = priv->plat->serdes_powerup(ndev,
	5553	priv->plat->bsp_priv);
	5554
	5555	if (ret < 0)
	5556	return ret;
	5557	}
	5558
	5559	if (!device_may_wakeup(priv->device) \|\| !priv->plat->pmt) {
	5560	rtnl_lock();
	5561	phylink_start(priv->phylink);
	5562	/* We may have called phylink_speed_down before */
	5563	phylink_speed_up(priv->phylink);
	5564	rtnl_unlock();
	5565	}
	5566
	5567	rtnl_lock();
	5568	mutex_lock(&priv->lock);
	5569
	5570	stmmac_reset_queues_param(priv);
	5571
	5572	stmmac_free_tx_skbufs(priv);
	5573	stmmac_clear_descriptors(priv);
	5574
	5575	stmmac_hw_setup(ndev, false);
	5576	stmmac_init_coalesce(priv);
	5577	stmmac_set_rx_mode(ndev);
	5578
	5579	stmmac_restore_hw_vlan_rx_fltr(priv, ndev, priv->hw);
	5580
	5581	stmmac_enable_all_queues(priv);
	5582
	5583	mutex_unlock(&priv->lock);
	5584	rtnl_unlock();
	5585
	5586	phylink_mac_change(priv->phylink, true);
	5587
	5588	netif_device_attach(ndev);
	5589
	5590	return 0;
	5591	}
	5592	EXPORT_SYMBOL_GPL(stmmac_resume);
	5593
	5594	#ifndef MODULE
	5595	static int __init stmmac_cmdline_opt(char *str)
	5596	{
	5597	char *opt;
	5598
	5599	if (!str \|\| !*str)
	5600	return 1;
	5601	while ((opt = strsep(&str, ",")) != NULL) {
	5602	if (!strncmp(opt, "debug:", 6)) {
	5603	if (kstrtoint(opt + 6, 0, &debug))
	5604	goto err;
	5605	} else if (!strncmp(opt, "phyaddr:", 8)) {
	5606	if (kstrtoint(opt + 8, 0, &phyaddr))
	5607	goto err;
	5608	} else if (!strncmp(opt, "buf_sz:", 7)) {
	5609	if (kstrtoint(opt + 7, 0, &buf_sz))
	5610	goto err;
	5611	} else if (!strncmp(opt, "tc:", 3)) {
	5612	if (kstrtoint(opt + 3, 0, &tc))
	5613	goto err;
	5614	} else if (!strncmp(opt, "watchdog:", 9)) {
	5615	if (kstrtoint(opt + 9, 0, &watchdog))
	5616	goto err;
	5617	} else if (!strncmp(opt, "flow_ctrl:", 10)) {
	5618	if (kstrtoint(opt + 10, 0, &flow_ctrl))
	5619	goto err;
	5620	} else if (!strncmp(opt, "pause:", 6)) {
	5621	if (kstrtoint(opt + 6, 0, &pause))
	5622	goto err;
	5623	} else if (!strncmp(opt, "eee_timer:", 10)) {
	5624	if (kstrtoint(opt + 10, 0, &eee_timer))
	5625	goto err;
	5626	} else if (!strncmp(opt, "chain_mode:", 11)) {
	5627	if (kstrtoint(opt + 11, 0, &chain_mode))
	5628	goto err;
	5629	}
	5630	}
	5631	return 1;
	5632
	5633	err:
	5634	pr_err("%s: ERROR broken module parameter conversion", __func__);
	5635	return 1;
	5636	}
	5637
	5638	__setup("stmmaceth=", stmmac_cmdline_opt);
	5639	#endif /* MODULE */
	5640
	5641	static int __init stmmac_init(void)
	5642	{
	5643	#ifdef CONFIG_DEBUG_FS
	5644	/* Create debugfs main directory if it doesn't exist yet */
	5645	if (!stmmac_fs_dir)
	5646	stmmac_fs_dir = debugfs_create_dir(STMMAC_RESOURCE_NAME, NULL);
	5647	register_netdevice_notifier(&stmmac_notifier);
	5648	#endif
	5649
	5650	return 0;
	5651	}
	5652
	5653	static void __exit stmmac_exit(void)
	5654	{
	5655	#ifdef CONFIG_DEBUG_FS
	5656	unregister_netdevice_notifier(&stmmac_notifier);
	5657	debugfs_remove_recursive(stmmac_fs_dir);
	5658	#endif
	5659	}
	5660
	5661	module_init(stmmac_init)
	5662	module_exit(stmmac_exit)
	5663
	5664	MODULE_DESCRIPTION("STMMAC 10/100/1000 Ethernet device driver");
	5665	MODULE_AUTHOR("Giuseppe Cavallaro <peppe.cavallaro@st.com>");
	5666	MODULE_LICENSE("GPL");