armada-37xx-periph.c

  1// SPDX-License-Identifier: GPL-2.0+
  2/*
  3 * Marvell Armada 37xx SoC Peripheral clocks
  4 *
  5 * Copyright (C) 2016 Marvell
  6 *
  7 * Gregory CLEMENT <gregory.clement@free-electrons.com>
  8 *
  9 * Most of the peripheral clocks can be modelled like this:
 10 *             _____    _______    _______
 11 * TBG-A-P  --|     |  |       |  |       |   ______
 12 * TBG-B-P  --| Mux |--| /div1 |--| /div2 |--| Gate |--> perip_clk
 13 * TBG-A-S  --|     |  |       |  |       |  |______|
 14 * TBG-B-S  --|_____|  |_______|  |_______|
 15 *
 16 * However some clocks may use only one or two block or and use the
 17 * xtal clock as parent.
 18 */
 19
 20#include <linux/clk-provider.h>
 21#include <linux/io.h>
 22#include <linux/mfd/syscon.h>
 23#include <linux/of.h>
 24#include <linux/platform_device.h>
 25#include <linux/regmap.h>
 26#include <linux/slab.h>
 27#include <linux/jiffies.h>
 28
 29#define TBG_SEL		0x0
 30#define DIV_SEL0	0x4
 31#define DIV_SEL1	0x8
 32#define DIV_SEL2	0xC
 33#define CLK_SEL		0x10
 34#define CLK_DIS		0x14
 35
 36#define  ARMADA_37XX_DVFS_LOAD_1 1
 37#define LOAD_LEVEL_NR	4
 38
 39#define ARMADA_37XX_NB_L0L1	0x18
 40#define ARMADA_37XX_NB_L2L3	0x1C
 41#define		ARMADA_37XX_NB_TBG_DIV_OFF	13
 42#define		ARMADA_37XX_NB_TBG_DIV_MASK	0x7
 43#define		ARMADA_37XX_NB_CLK_SEL_OFF	11
 44#define		ARMADA_37XX_NB_CLK_SEL_MASK	0x1
 45#define		ARMADA_37XX_NB_TBG_SEL_OFF	9
 46#define		ARMADA_37XX_NB_TBG_SEL_MASK	0x3
 47#define		ARMADA_37XX_NB_CONFIG_SHIFT	16
 48#define ARMADA_37XX_NB_DYN_MOD	0x24
 49#define		ARMADA_37XX_NB_DFS_EN	31
 50#define ARMADA_37XX_NB_CPU_LOAD	0x30
 51#define		ARMADA_37XX_NB_CPU_LOAD_MASK	0x3
 52#define		ARMADA_37XX_DVFS_LOAD_0		0
 53#define		ARMADA_37XX_DVFS_LOAD_1		1
 54#define		ARMADA_37XX_DVFS_LOAD_2		2
 55#define		ARMADA_37XX_DVFS_LOAD_3		3
 56
 57struct clk_periph_driver_data {
 58	struct clk_hw_onecell_data *hw_data;
 59	spinlock_t lock;
 60	void __iomem *reg;
 61
 62	/* Storage registers for suspend/resume operations */
 63	u32 tbg_sel;
 64	u32 div_sel0;
 65	u32 div_sel1;
 66	u32 div_sel2;
 67	u32 clk_sel;
 68	u32 clk_dis;
 69};
 70
 71struct clk_double_div {
 72	struct clk_hw hw;
 73	void __iomem *reg1;
 74	u8 shift1;
 75	void __iomem *reg2;
 76	u8 shift2;
 77};
 78
 79struct clk_pm_cpu {
 80	struct clk_hw hw;
 81	void __iomem *reg_mux;
 82	u8 shift_mux;
 83	u32 mask_mux;
 84	void __iomem *reg_div;
 85	u8 shift_div;
 86	struct regmap *nb_pm_base;
 87	unsigned long l1_expiration;
 88};
 89
 90#define to_clk_double_div(_hw) container_of(_hw, struct clk_double_div, hw)
 91#define to_clk_pm_cpu(_hw) container_of(_hw, struct clk_pm_cpu, hw)
 92
 93struct clk_periph_data {
 94	const char *name;
 95	const char * const *parent_names;
 96	int num_parents;
 97	struct clk_hw *mux_hw;
 98	struct clk_hw *rate_hw;
 99	struct clk_hw *gate_hw;
100	struct clk_hw *muxrate_hw;
101	bool is_double_div;
102};
103
104static const struct clk_div_table clk_table6[] = {
105	{ .val = 1, .div = 1, },
106	{ .val = 2, .div = 2, },
107	{ .val = 3, .div = 3, },
108	{ .val = 4, .div = 4, },
109	{ .val = 5, .div = 5, },
110	{ .val = 6, .div = 6, },
111	{ .val = 0, .div = 0, }, /* last entry */
112};
113
114static const struct clk_div_table clk_table1[] = {
115	{ .val = 0, .div = 1, },
116	{ .val = 1, .div = 2, },
117	{ .val = 0, .div = 0, }, /* last entry */
118};
119
120static const struct clk_div_table clk_table2[] = {
121	{ .val = 0, .div = 2, },
122	{ .val = 1, .div = 4, },
123	{ .val = 0, .div = 0, }, /* last entry */
124};
125
126static const struct clk_ops clk_double_div_ops;
127static const struct clk_ops clk_pm_cpu_ops;
128
129#define PERIPH_GATE(_name, _bit)		\
130struct clk_gate gate_##_name = {		\
131	.reg = (void *)CLK_DIS,			\
132	.bit_idx = _bit,			\
133	.hw.init = &(struct clk_init_data){	\
134		.ops =  &clk_gate_ops,		\
135	}					\
136};
137
138#define PERIPH_MUX(_name, _shift)		\
139struct clk_mux mux_##_name = {			\
140	.reg = (void *)TBG_SEL,			\
141	.shift = _shift,			\
142	.mask = 3,				\
143	.hw.init = &(struct clk_init_data){	\
144		.ops =  &clk_mux_ro_ops,	\
145	}					\
146};
147
148#define PERIPH_DOUBLEDIV(_name, _reg1, _reg2, _shift1, _shift2)	\
149struct clk_double_div rate_##_name = {		\
150	.reg1 = (void *)_reg1,			\
151	.reg2 = (void *)_reg2,			\
152	.shift1 = _shift1,			\
153	.shift2 = _shift2,			\
154	.hw.init = &(struct clk_init_data){	\
155		.ops =  &clk_double_div_ops,	\
156	}					\
157};
158
159#define PERIPH_DIV(_name, _reg, _shift, _table)	\
160struct clk_divider rate_##_name = {		\
161	.reg = (void *)_reg,			\
162	.table = _table,			\
163	.shift = _shift,			\
164	.hw.init = &(struct clk_init_data){	\
165		.ops =  &clk_divider_ro_ops,	\
166	}					\
167};
168
169#define PERIPH_PM_CPU(_name, _shift1, _reg, _shift2)	\
170struct clk_pm_cpu muxrate_##_name = {		\
171	.reg_mux = (void *)TBG_SEL,		\
172	.mask_mux = 3,				\
173	.shift_mux = _shift1,			\
174	.reg_div = (void *)_reg,		\
175	.shift_div = _shift2,			\
176	.hw.init = &(struct clk_init_data){	\
177		.ops =  &clk_pm_cpu_ops,	\
178	}					\
179};
180
181#define PERIPH_CLK_FULL_DD(_name, _bit, _shift, _reg1, _reg2, _shift1, _shift2)\
182static PERIPH_GATE(_name, _bit);			    \
183static PERIPH_MUX(_name, _shift);			    \
184static PERIPH_DOUBLEDIV(_name, _reg1, _reg2, _shift1, _shift2);
185
186#define PERIPH_CLK_FULL(_name, _bit, _shift, _reg, _shift1, _table)	\
187static PERIPH_GATE(_name, _bit);			    \
188static PERIPH_MUX(_name, _shift);			    \
189static PERIPH_DIV(_name, _reg, _shift1, _table);
190
191#define PERIPH_CLK_GATE_DIV(_name, _bit,  _reg, _shift, _table)	\
192static PERIPH_GATE(_name, _bit);			\
193static PERIPH_DIV(_name, _reg, _shift, _table);
194
195#define PERIPH_CLK_MUX_DD(_name, _shift, _reg1, _reg2, _shift1, _shift2)\
196static PERIPH_MUX(_name, _shift);			    \
197static PERIPH_DOUBLEDIV(_name, _reg1, _reg2, _shift1, _shift2);
198
199#define REF_CLK_FULL(_name)				\
200	{ .name = #_name,				\
201	  .parent_names = (const char *[]){ "TBG-A-P",	\
202	      "TBG-B-P", "TBG-A-S", "TBG-B-S"},		\
203	  .num_parents = 4,				\
204	  .mux_hw = &mux_##_name.hw,			\
205	  .gate_hw = &gate_##_name.hw,			\
206	  .rate_hw = &rate_##_name.hw,			\
207	}
208
209#define REF_CLK_FULL_DD(_name)				\
210	{ .name = #_name,				\
211	  .parent_names = (const char *[]){ "TBG-A-P",	\
212	      "TBG-B-P", "TBG-A-S", "TBG-B-S"},		\
213	  .num_parents = 4,				\
214	  .mux_hw = &mux_##_name.hw,			\
215	  .gate_hw = &gate_##_name.hw,			\
216	  .rate_hw = &rate_##_name.hw,			\
217	  .is_double_div = true,			\
218	}
219
220#define REF_CLK_GATE(_name, _parent_name)			\
221	{ .name = #_name,					\
222	  .parent_names = (const char *[]){ _parent_name},	\
223	  .num_parents = 1,					\
224	  .gate_hw = &gate_##_name.hw,				\
225	}
226
227#define REF_CLK_GATE_DIV(_name, _parent_name)			\
228	{ .name = #_name,					\
229	  .parent_names = (const char *[]){ _parent_name},	\
230	  .num_parents = 1,					\
231	  .gate_hw = &gate_##_name.hw,				\
232	  .rate_hw = &rate_##_name.hw,				\
233	}
234
235#define REF_CLK_PM_CPU(_name)				\
236	{ .name = #_name,				\
237	  .parent_names = (const char *[]){ "TBG-A-P",	\
238	      "TBG-B-P", "TBG-A-S", "TBG-B-S"},		\
239	  .num_parents = 4,				\
240	  .muxrate_hw = &muxrate_##_name.hw,		\
241	}
242
243#define REF_CLK_MUX_DD(_name)				\
244	{ .name = #_name,				\
245	  .parent_names = (const char *[]){ "TBG-A-P",	\
246	      "TBG-B-P", "TBG-A-S", "TBG-B-S"},		\
247	  .num_parents = 4,				\
248	  .mux_hw = &mux_##_name.hw,			\
249	  .rate_hw = &rate_##_name.hw,			\
250	  .is_double_div = true,			\
251	}
252
253/* NB periph clocks */
254PERIPH_CLK_FULL_DD(mmc, 2, 0, DIV_SEL2, DIV_SEL2, 16, 13);
255PERIPH_CLK_FULL_DD(sata_host, 3, 2, DIV_SEL2, DIV_SEL2, 10, 7);
256PERIPH_CLK_FULL_DD(sec_at, 6, 4, DIV_SEL1, DIV_SEL1, 3, 0);
257PERIPH_CLK_FULL_DD(sec_dap, 7, 6, DIV_SEL1, DIV_SEL1, 9, 6);
258PERIPH_CLK_FULL_DD(tscem, 8, 8, DIV_SEL1, DIV_SEL1, 15, 12);
259PERIPH_CLK_FULL(tscem_tmx, 10, 10, DIV_SEL1, 18, clk_table6);
260static PERIPH_GATE(avs, 11);
261PERIPH_CLK_FULL_DD(pwm, 13, 14, DIV_SEL0, DIV_SEL0, 3, 0);
262PERIPH_CLK_FULL_DD(sqf, 12, 12, DIV_SEL1, DIV_SEL1, 27, 24);
263static PERIPH_GATE(i2c_2, 16);
264static PERIPH_GATE(i2c_1, 17);
265PERIPH_CLK_GATE_DIV(ddr_phy, 19, DIV_SEL0, 18, clk_table2);
266PERIPH_CLK_FULL_DD(ddr_fclk, 21, 16, DIV_SEL0, DIV_SEL0, 15, 12);
267PERIPH_CLK_FULL(trace, 22, 18, DIV_SEL0, 20, clk_table6);
268PERIPH_CLK_FULL(counter, 23, 20, DIV_SEL0, 23, clk_table6);
269PERIPH_CLK_FULL_DD(eip97, 24, 24, DIV_SEL2, DIV_SEL2, 22, 19);
270static PERIPH_PM_CPU(cpu, 22, DIV_SEL0, 28);
271
272static struct clk_periph_data data_nb[] = {
273	REF_CLK_FULL_DD(mmc),
274	REF_CLK_FULL_DD(sata_host),
275	REF_CLK_FULL_DD(sec_at),
276	REF_CLK_FULL_DD(sec_dap),
277	REF_CLK_FULL_DD(tscem),
278	REF_CLK_FULL(tscem_tmx),
279	REF_CLK_GATE(avs, "xtal"),
280	REF_CLK_FULL_DD(sqf),
281	REF_CLK_FULL_DD(pwm),
282	REF_CLK_GATE(i2c_2, "xtal"),
283	REF_CLK_GATE(i2c_1, "xtal"),
284	REF_CLK_GATE_DIV(ddr_phy, "TBG-A-S"),
285	REF_CLK_FULL_DD(ddr_fclk),
286	REF_CLK_FULL(trace),
287	REF_CLK_FULL(counter),
288	REF_CLK_FULL_DD(eip97),
289	REF_CLK_PM_CPU(cpu),
290	{ },
291};
292
293/* SB periph clocks */
294PERIPH_CLK_MUX_DD(gbe_50, 6, DIV_SEL2, DIV_SEL2, 6, 9);
295PERIPH_CLK_MUX_DD(gbe_core, 8, DIV_SEL1, DIV_SEL1, 18, 21);
296PERIPH_CLK_MUX_DD(gbe_125, 10, DIV_SEL1, DIV_SEL1, 6, 9);
297static PERIPH_GATE(gbe1_50, 0);
298static PERIPH_GATE(gbe0_50, 1);
299static PERIPH_GATE(gbe1_125, 2);
300static PERIPH_GATE(gbe0_125, 3);
301PERIPH_CLK_GATE_DIV(gbe1_core, 4, DIV_SEL1, 13, clk_table1);
302PERIPH_CLK_GATE_DIV(gbe0_core, 5, DIV_SEL1, 14, clk_table1);
303PERIPH_CLK_GATE_DIV(gbe_bm, 12, DIV_SEL1, 0, clk_table1);
304PERIPH_CLK_FULL_DD(sdio, 11, 14, DIV_SEL0, DIV_SEL0, 3, 6);
305PERIPH_CLK_FULL_DD(usb32_usb2_sys, 16, 16, DIV_SEL0, DIV_SEL0, 9, 12);
306PERIPH_CLK_FULL_DD(usb32_ss_sys, 17, 18, DIV_SEL0, DIV_SEL0, 15, 18);
307static PERIPH_GATE(pcie, 14);
308
309static struct clk_periph_data data_sb[] = {
310	REF_CLK_MUX_DD(gbe_50),
311	REF_CLK_MUX_DD(gbe_core),
312	REF_CLK_MUX_DD(gbe_125),
313	REF_CLK_GATE(gbe1_50, "gbe_50"),
314	REF_CLK_GATE(gbe0_50, "gbe_50"),
315	REF_CLK_GATE(gbe1_125, "gbe_125"),
316	REF_CLK_GATE(gbe0_125, "gbe_125"),
317	REF_CLK_GATE_DIV(gbe1_core, "gbe_core"),
318	REF_CLK_GATE_DIV(gbe0_core, "gbe_core"),
319	REF_CLK_GATE_DIV(gbe_bm, "gbe_core"),
320	REF_CLK_FULL_DD(sdio),
321	REF_CLK_FULL_DD(usb32_usb2_sys),
322	REF_CLK_FULL_DD(usb32_ss_sys),
323	REF_CLK_GATE(pcie, "gbe_core"),
324	{ },
325};
326
327static unsigned int get_div(void __iomem *reg, int shift)
328{
329	u32 val;
330
331	val = (readl(reg) >> shift) & 0x7;
332	if (val > 6)
333		return 0;
334	return val;
335}
336
337static unsigned long clk_double_div_recalc_rate(struct clk_hw *hw,
338						unsigned long parent_rate)
339{
340	struct clk_double_div *double_div = to_clk_double_div(hw);
341	unsigned int div;
342
343	div = get_div(double_div->reg1, double_div->shift1);
344	div *= get_div(double_div->reg2, double_div->shift2);
345
346	return DIV_ROUND_UP_ULL((u64)parent_rate, div);
347}
348
349static const struct clk_ops clk_double_div_ops = {
350	.recalc_rate = clk_double_div_recalc_rate,
351};
352
353static void armada_3700_pm_dvfs_update_regs(unsigned int load_level,
354					    unsigned int *reg,
355					    unsigned int *offset)
356{
357	if (load_level <= ARMADA_37XX_DVFS_LOAD_1)
358		*reg = ARMADA_37XX_NB_L0L1;
359	else
360		*reg = ARMADA_37XX_NB_L2L3;
361
362	if (load_level == ARMADA_37XX_DVFS_LOAD_0 ||
363	    load_level ==  ARMADA_37XX_DVFS_LOAD_2)
364		*offset += ARMADA_37XX_NB_CONFIG_SHIFT;
365}
366
367static bool armada_3700_pm_dvfs_is_enabled(struct regmap *base)
368{
369	unsigned int val, reg = ARMADA_37XX_NB_DYN_MOD;
370
371	if (IS_ERR(base))
372		return false;
373
374	regmap_read(base, reg, &val);
375
376	return !!(val & BIT(ARMADA_37XX_NB_DFS_EN));
377}
378
379static unsigned int armada_3700_pm_dvfs_get_cpu_div(struct regmap *base)
380{
381	unsigned int reg = ARMADA_37XX_NB_CPU_LOAD;
382	unsigned int offset = ARMADA_37XX_NB_TBG_DIV_OFF;
383	unsigned int load_level, div;
384
385	/*
386	 * This function is always called after the function
387	 * armada_3700_pm_dvfs_is_enabled, so no need to check again
388	 * if the base is valid.
389	 */
390	regmap_read(base, reg, &load_level);
391
392	/*
393	 * The register and the offset inside this register accessed to
394	 * read the current divider depend on the load level
395	 */
396	load_level &= ARMADA_37XX_NB_CPU_LOAD_MASK;
397	armada_3700_pm_dvfs_update_regs(load_level, &reg, &offset);
398
399	regmap_read(base, reg, &div);
400
401	return (div >> offset) & ARMADA_37XX_NB_TBG_DIV_MASK;
402}
403
404static unsigned int armada_3700_pm_dvfs_get_cpu_parent(struct regmap *base)
405{
406	unsigned int reg = ARMADA_37XX_NB_CPU_LOAD;
407	unsigned int offset = ARMADA_37XX_NB_TBG_SEL_OFF;
408	unsigned int load_level, sel;
409
410	/*
411	 * This function is always called after the function
412	 * armada_3700_pm_dvfs_is_enabled, so no need to check again
413	 * if the base is valid
414	 */
415	regmap_read(base, reg, &load_level);
416
417	/*
418	 * The register and the offset inside this register accessed to
419	 * read the current divider depend on the load level
420	 */
421	load_level &= ARMADA_37XX_NB_CPU_LOAD_MASK;
422	armada_3700_pm_dvfs_update_regs(load_level, &reg, &offset);
423
424	regmap_read(base, reg, &sel);
425
426	return (sel >> offset) & ARMADA_37XX_NB_TBG_SEL_MASK;
427}
428
429static u8 clk_pm_cpu_get_parent(struct clk_hw *hw)
430{
431	struct clk_pm_cpu *pm_cpu = to_clk_pm_cpu(hw);
432	u32 val;
433
434	if (armada_3700_pm_dvfs_is_enabled(pm_cpu->nb_pm_base)) {
435		val = armada_3700_pm_dvfs_get_cpu_parent(pm_cpu->nb_pm_base);
436	} else {
437		val = readl(pm_cpu->reg_mux) >> pm_cpu->shift_mux;
438		val &= pm_cpu->mask_mux;
439	}
440
441	return val;
442}
443
444static unsigned long clk_pm_cpu_recalc_rate(struct clk_hw *hw,
445					    unsigned long parent_rate)
446{
447	struct clk_pm_cpu *pm_cpu = to_clk_pm_cpu(hw);
448	unsigned int div;
449
450	if (armada_3700_pm_dvfs_is_enabled(pm_cpu->nb_pm_base))
451		div = armada_3700_pm_dvfs_get_cpu_div(pm_cpu->nb_pm_base);
452	else
453		div = get_div(pm_cpu->reg_div, pm_cpu->shift_div);
454	return DIV_ROUND_UP_ULL((u64)parent_rate, div);
455}
456
457static int clk_pm_cpu_determine_rate(struct clk_hw *hw,
458				     struct clk_rate_request *req)
459{
460	struct clk_pm_cpu *pm_cpu = to_clk_pm_cpu(hw);
461	struct regmap *base = pm_cpu->nb_pm_base;
462	unsigned int div = req->best_parent_rate / req->rate;
463	unsigned int load_level;
464	/* only available when DVFS is enabled */
465	if (!armada_3700_pm_dvfs_is_enabled(base))
466		return -EINVAL;
467
468	for (load_level = 0; load_level < LOAD_LEVEL_NR; load_level++) {
469		unsigned int reg, val, offset = ARMADA_37XX_NB_TBG_DIV_OFF;
470
471		armada_3700_pm_dvfs_update_regs(load_level, &reg, &offset);
472
473		regmap_read(base, reg, &val);
474
475		val >>= offset;
476		val &= ARMADA_37XX_NB_TBG_DIV_MASK;
477		if (val == div) {
478			/*
479			 * We found a load level matching the target
480			 * divider, switch to this load level and
481			 * return.
482			 */
483			req->rate = req->best_parent_rate / div;
484
485			return 0;
486		}
487	}
488
489	/* We didn't find any valid divider */
490	return -EINVAL;
491}
492
493/*
494 * Workaround when base CPU frequnecy is 1000 or 1200 MHz
495 *
496 * Switching the CPU from the L2 or L3 frequencies (250/300 or 200 MHz
497 * respectively) to L0 frequency (1/1.2 GHz) requires a significant
498 * amount of time to let VDD stabilize to the appropriate
499 * voltage. This amount of time is large enough that it cannot be
500 * covered by the hardware countdown register. Due to this, the CPU
501 * might start operating at L0 before the voltage is stabilized,
502 * leading to CPU stalls.
503 *
504 * To work around this problem, we prevent switching directly from the
505 * L2/L3 frequencies to the L0 frequency, and instead switch to the L1
506 * frequency in-between. The sequence therefore becomes:
507 * 1. First switch from L2/L3 (200/250/300 MHz) to L1 (500/600 MHz)
508 * 2. Sleep 20ms for stabling VDD voltage
509 * 3. Then switch from L1 (500/600 MHz) to L0 (1000/1200 MHz).
510 */
511static void clk_pm_cpu_set_rate_wa(struct clk_pm_cpu *pm_cpu,
512				   unsigned int new_level, unsigned long rate,
513				   struct regmap *base)
514{
515	unsigned int cur_level;
516
517	regmap_read(base, ARMADA_37XX_NB_CPU_LOAD, &cur_level);
518	cur_level &= ARMADA_37XX_NB_CPU_LOAD_MASK;
519
520	if (cur_level == new_level)
521		return;
522
523	/*
524	 * System wants to go to L1 on its own. If we are going from L2/L3,
525	 * remember when 20ms will expire. If from L0, set the value so that
526	 * next switch to L0 won't have to wait.
527	 */
528	if (new_level == ARMADA_37XX_DVFS_LOAD_1) {
529		if (cur_level == ARMADA_37XX_DVFS_LOAD_0)
530			pm_cpu->l1_expiration = jiffies;
531		else
532			pm_cpu->l1_expiration = jiffies + msecs_to_jiffies(20);
533		return;
534	}
535
536	/*
537	 * If we are setting to L2/L3, just invalidate L1 expiration time,
538	 * sleeping is not needed.
539	 */
540	if (rate < 1000*1000*1000)
541		goto invalidate_l1_exp;
542
543	/*
544	 * We are going to L0 with rate >= 1GHz. Check whether we have been at
545	 * L1 for long enough time. If not, go to L1 for 20ms.
546	 */
547	if (pm_cpu->l1_expiration && time_is_before_eq_jiffies(pm_cpu->l1_expiration))
548		goto invalidate_l1_exp;
549
550	regmap_update_bits(base, ARMADA_37XX_NB_CPU_LOAD,
551			   ARMADA_37XX_NB_CPU_LOAD_MASK,
552			   ARMADA_37XX_DVFS_LOAD_1);
553	msleep(20);
554
555invalidate_l1_exp:
556	pm_cpu->l1_expiration = 0;
557}
558
559static int clk_pm_cpu_set_rate(struct clk_hw *hw, unsigned long rate,
560			       unsigned long parent_rate)
561{
562	struct clk_pm_cpu *pm_cpu = to_clk_pm_cpu(hw);
563	struct regmap *base = pm_cpu->nb_pm_base;
564	unsigned int div = parent_rate / rate;
565	unsigned int load_level;
566
567	/* only available when DVFS is enabled */
568	if (!armada_3700_pm_dvfs_is_enabled(base))
569		return -EINVAL;
570
571	for (load_level = 0; load_level < LOAD_LEVEL_NR; load_level++) {
572		unsigned int reg, mask, val,
573			offset = ARMADA_37XX_NB_TBG_DIV_OFF;
574
575		armada_3700_pm_dvfs_update_regs(load_level, &reg, &offset);
576
577		regmap_read(base, reg, &val);
578		val >>= offset;
579		val &= ARMADA_37XX_NB_TBG_DIV_MASK;
580
581		if (val == div) {
582			/*
583			 * We found a load level matching the target
584			 * divider, switch to this load level and
585			 * return.
586			 */
587			reg = ARMADA_37XX_NB_CPU_LOAD;
588			mask = ARMADA_37XX_NB_CPU_LOAD_MASK;
589
590			/* Apply workaround when base CPU frequency is 1000 or 1200 MHz */
591			if (parent_rate >= 1000*1000*1000)
592				clk_pm_cpu_set_rate_wa(pm_cpu, load_level, rate, base);
593
594			regmap_update_bits(base, reg, mask, load_level);
595
596			return rate;
597		}
598	}
599
600	/* We didn't find any valid divider */
601	return -EINVAL;
602}
603
604static const struct clk_ops clk_pm_cpu_ops = {
605	.get_parent = clk_pm_cpu_get_parent,
606	.determine_rate = clk_pm_cpu_determine_rate,
607	.set_rate = clk_pm_cpu_set_rate,
608	.recalc_rate = clk_pm_cpu_recalc_rate,
609};
610
611static const struct of_device_id armada_3700_periph_clock_of_match[] = {
612	{ .compatible = "marvell,armada-3700-periph-clock-nb",
613	  .data = data_nb, },
614	{ .compatible = "marvell,armada-3700-periph-clock-sb",
615	.data = data_sb, },
616	{ }
617};
618
619static int armada_3700_add_composite_clk(const struct clk_periph_data *data,
620					 void __iomem *reg, spinlock_t *lock,
621					 struct device *dev, struct clk_hw **hw)
622{
623	const struct clk_ops *mux_ops = NULL, *gate_ops = NULL,
624		*rate_ops = NULL;
625	struct clk_hw *mux_hw = NULL, *gate_hw = NULL, *rate_hw = NULL;
626
627	if (data->mux_hw) {
628		struct clk_mux *mux;
629
630		mux_hw = data->mux_hw;
631		mux = to_clk_mux(mux_hw);
632		mux->lock = lock;
633		mux_ops = mux_hw->init->ops;
634		mux->reg = reg + (u64)mux->reg;
635	}
636
637	if (data->gate_hw) {
638		struct clk_gate *gate;
639
640		gate_hw = data->gate_hw;
641		gate = to_clk_gate(gate_hw);
642		gate->lock = lock;
643		gate_ops = gate_hw->init->ops;
644		gate->reg = reg + (u64)gate->reg;
645		gate->flags = CLK_GATE_SET_TO_DISABLE;
646	}
647
648	if (data->rate_hw) {
649		rate_hw = data->rate_hw;
650		rate_ops = rate_hw->init->ops;
651		if (data->is_double_div) {
652			struct clk_double_div *rate;
653
654			rate =  to_clk_double_div(rate_hw);
655			rate->reg1 = reg + (u64)rate->reg1;
656			rate->reg2 = reg + (u64)rate->reg2;
657		} else {
658			struct clk_divider *rate = to_clk_divider(rate_hw);
659			const struct clk_div_table *clkt;
660			int table_size = 0;
661
662			rate->reg = reg + (u64)rate->reg;
663			for (clkt = rate->table; clkt->div; clkt++)
664				table_size++;
665			rate->width = order_base_2(table_size);
666			rate->lock = lock;
667		}
668	}
669
670	if (data->muxrate_hw) {
671		struct clk_pm_cpu *pmcpu_clk;
672		struct clk_hw *muxrate_hw = data->muxrate_hw;
673		struct regmap *map;
674
675		pmcpu_clk =  to_clk_pm_cpu(muxrate_hw);
676		pmcpu_clk->reg_mux = reg + (u64)pmcpu_clk->reg_mux;
677		pmcpu_clk->reg_div = reg + (u64)pmcpu_clk->reg_div;
678
679		mux_hw = muxrate_hw;
680		rate_hw = muxrate_hw;
681		mux_ops = muxrate_hw->init->ops;
682		rate_ops = muxrate_hw->init->ops;
683
684		map = syscon_regmap_lookup_by_compatible(
685				"marvell,armada-3700-nb-pm");
686		pmcpu_clk->nb_pm_base = map;
687	}
688
689	*hw = clk_hw_register_composite(dev, data->name, data->parent_names,
690					data->num_parents, mux_hw,
691					mux_ops, rate_hw, rate_ops,
692					gate_hw, gate_ops, CLK_IGNORE_UNUSED);
693
694	return PTR_ERR_OR_ZERO(*hw);
695}
696
697static int __maybe_unused armada_3700_periph_clock_suspend(struct device *dev)
698{
699	struct clk_periph_driver_data *data = dev_get_drvdata(dev);
700
701	data->tbg_sel = readl(data->reg + TBG_SEL);
702	data->div_sel0 = readl(data->reg + DIV_SEL0);
703	data->div_sel1 = readl(data->reg + DIV_SEL1);
704	data->div_sel2 = readl(data->reg + DIV_SEL2);
705	data->clk_sel = readl(data->reg + CLK_SEL);
706	data->clk_dis = readl(data->reg + CLK_DIS);
707
708	return 0;
709}
710
711static int __maybe_unused armada_3700_periph_clock_resume(struct device *dev)
712{
713	struct clk_periph_driver_data *data = dev_get_drvdata(dev);
714
715	/* Follow the same order than what the Cortex-M3 does (ATF code) */
716	writel(data->clk_dis, data->reg + CLK_DIS);
717	writel(data->div_sel0, data->reg + DIV_SEL0);
718	writel(data->div_sel1, data->reg + DIV_SEL1);
719	writel(data->div_sel2, data->reg + DIV_SEL2);
720	writel(data->tbg_sel, data->reg + TBG_SEL);
721	writel(data->clk_sel, data->reg + CLK_SEL);
722
723	return 0;
724}
725
726static const struct dev_pm_ops armada_3700_periph_clock_pm_ops = {
727	SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(armada_3700_periph_clock_suspend,
728				      armada_3700_periph_clock_resume)
729};
730
731static int armada_3700_periph_clock_probe(struct platform_device *pdev)
732{
733	struct clk_periph_driver_data *driver_data;
734	struct device_node *np = pdev->dev.of_node;
735	const struct clk_periph_data *data;
736	struct device *dev = &pdev->dev;
737	int num_periph = 0, i, ret;
738
739	data = of_device_get_match_data(dev);
740	if (!data)
741		return -ENODEV;
742
743	while (data[num_periph].name)
744		num_periph++;
745
746	driver_data = devm_kzalloc(dev, sizeof(*driver_data), GFP_KERNEL);
747	if (!driver_data)
748		return -ENOMEM;
749
750	driver_data->hw_data = devm_kzalloc(dev,
751					    struct_size(driver_data->hw_data,
752							hws, num_periph),
753					    GFP_KERNEL);
754	if (!driver_data->hw_data)
755		return -ENOMEM;
756	driver_data->hw_data->num = num_periph;
757
758	driver_data->reg = devm_platform_ioremap_resource(pdev, 0);
759	if (IS_ERR(driver_data->reg))
760		return PTR_ERR(driver_data->reg);
761
762	spin_lock_init(&driver_data->lock);
763
764	for (i = 0; i < num_periph; i++) {
765		struct clk_hw **hw = &driver_data->hw_data->hws[i];
766		if (armada_3700_add_composite_clk(&data[i], driver_data->reg,
767						  &driver_data->lock, dev, hw))
768			dev_err(dev, "Can't register periph clock %s\n",
769				data[i].name);
770	}
771
772	ret = of_clk_add_hw_provider(np, of_clk_hw_onecell_get,
773				     driver_data->hw_data);
774	if (ret) {
775		for (i = 0; i < num_periph; i++)
776			clk_hw_unregister(driver_data->hw_data->hws[i]);
777		return ret;
778	}
779
780	platform_set_drvdata(pdev, driver_data);
781	return 0;
782}
783
784static void armada_3700_periph_clock_remove(struct platform_device *pdev)
785{
786	struct clk_periph_driver_data *data = platform_get_drvdata(pdev);
787	struct clk_hw_onecell_data *hw_data = data->hw_data;
788	int i;
789
790	of_clk_del_provider(pdev->dev.of_node);
791
792	for (i = 0; i < hw_data->num; i++)
793		clk_hw_unregister(hw_data->hws[i]);
794}
795
796static struct platform_driver armada_3700_periph_clock_driver = {
797	.probe = armada_3700_periph_clock_probe,
798	.remove = armada_3700_periph_clock_remove,
799	.driver		= {
800		.name	= "marvell-armada-3700-periph-clock",
801		.of_match_table = armada_3700_periph_clock_of_match,
802		.pm	= &armada_3700_periph_clock_pm_ops,
803	},
804};
805
806builtin_platform_driver(armada_3700_periph_clock_driver);