clk-krait.c

// SPDX-License-Identifier: GPL-2.0
// Copyright (c) 2018, The Linux Foundation. All rights reserved.

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/io.h>
#include <linux/delay.h>
#include <linux/err.h>
#include <linux/clk-provider.h>
#include <linux/spinlock.h>

#include <asm/krait-l2-accessors.h>

#include "clk-krait.h"

/* Secondary and primary muxes share the same cp15 register */
static DEFINE_SPINLOCK(krait_clock_reg_lock);

#define LPL_SHIFT	8
#define SECCLKAGD	BIT(4)

static void __krait_mux_set_sel(struct krait_mux_clk *mux, int sel)
{
	unsigned long flags;
	u32 regval;

	spin_lock_irqsave(&krait_clock_reg_lock, flags);


	regval = krait_get_l2_indirect_reg(mux->offset);

	pr_info("ASKED TO SWITCH TO SEL offset %x off %d regval %x\n", mux->offset, sel, regval );

	/* apq/ipq8064 Errata: disable sec_src clock gating during switch. */
	if (mux->disable_sec_src_gating) {
		regval |= SECCLKAGD;
		krait_set_l2_indirect_reg(mux->offset, regval);
	}

	regval &= ~(mux->mask << mux->shift);
	regval |= (sel & mux->mask) << mux->shift;
	if (mux->lpl) {
		regval &= ~(mux->mask << (mux->shift + LPL_SHIFT));
		regval |= (sel & mux->mask) << (mux->shift + LPL_SHIFT);
	}
	krait_set_l2_indirect_reg(mux->offset, regval);

	/* apq/ipq8064 Errata: re-enabled sec_src clock gating. */
	if (mux->disable_sec_src_gating) {
		regval &= ~SECCLKAGD;
		krait_set_l2_indirect_reg(mux->offset, regval);
	}

	/* Wait for switch to complete. */
	mb();
	udelay(1);

	/*
	 * Unlock now to make sure the mux register is not
	 * modified while switching to the new parent.
	 */
	spin_unlock_irqrestore(&krait_clock_reg_lock, flags);
}

static int krait_mux_set_parent(struct clk_hw *hw, u8 index)
{
	struct krait_mux_clk *mux = to_krait_mux_clk(hw);
	u32 sel;

	sel = clk_mux_index_to_val(mux->parent_map, 0, index);
	mux->en_mask = sel;
	/* Don't touch mux if CPU is off as it won't work */
	if (__clk_is_enabled(hw->clk))
		__krait_mux_set_sel(mux, sel);

	mux->reparent = true;

	return 0;
}

static u8 krait_mux_get_parent(struct clk_hw *hw)
{
	struct krait_mux_clk *mux = to_krait_mux_clk(hw);
	u32 sel;

	sel = krait_get_l2_indirect_reg(mux->offset);
	sel >>= mux->shift;
	sel &= mux->mask;
	mux->en_mask = sel;

	return clk_mux_val_to_index(hw, mux->parent_map, 0, sel);
}

const struct clk_ops krait_mux_clk_ops = {
	.set_parent = krait_mux_set_parent,
	.get_parent = krait_mux_get_parent,
	.determine_rate = __clk_mux_determine_rate_closest,
};
EXPORT_SYMBOL_GPL(krait_mux_clk_ops);

/* The divider can divide by 2, 4, 6 and 8. But we only really need div-2. */
static long krait_div2_round_rate(struct clk_hw *hw, unsigned long rate,
				  unsigned long *parent_rate)
{
	*parent_rate = clk_hw_round_rate(clk_hw_get_parent(hw), rate * 2);

	pr_info("DIV2 ROUND RATE rate %d parent rate %d rounded %d\n", rate, *parent_rate, DIV_ROUND_UP(*parent_rate, 2));

	return DIV_ROUND_UP(*parent_rate, 2);
}

static int krait_div2_set_rate(struct clk_hw *hw, unsigned long rate,
			       unsigned long parent_rate)
{
	struct krait_div2_clk *d = to_krait_div2_clk(hw);
	unsigned long flags;
	u32 regval;

	pr_info("DIV 2 SET RATE rate %d parent rate %d\n", rate, parent_rate);

	spin_lock_irqsave(&krait_clock_reg_lock, flags);
	regval = krait_get_l2_indirect_reg(d->offset);
	pr_info("DIV 2 SET RATE offset %x regval %x BEFORE\n", d->offset, regval);

	regval &= ~(d->mask << d->shift);
	regval |= (d->divisor & d->mask) << d->shift;

	if (d->lpl) {
		regval &= ~(d->mask << (d->shift + LPL_SHIFT));
		regval |= (d->divisor & d->mask) << (d->shift + LPL_SHIFT);
	}

	pr_info("DIV 2 SET RATE offset %x regval %x AFTER\n", d->offset, regval);

	krait_set_l2_indirect_reg(d->offset, regval);
	spin_unlock_irqrestore(&krait_clock_reg_lock, flags);

	return 0;
}

static unsigned long
krait_div2_recalc_rate(struct clk_hw *hw, unsigned long parent_rate)
{
	struct krait_div2_clk *d = to_krait_div2_clk(hw);
	u32 div;

	div = krait_get_l2_indirect_reg(d->offset);
	div >>= d->shift;
	div &= d->mask;

	pr_info("DIV 2 RECALC RATE %x\n",div);
	div = (div + 1) * 2;

	pr_info("DIV 2 RECAL RATE rate %d parent rate %d div %d\n", DIV_ROUND_UP(parent_rate, div), parent_rate, div);

	return DIV_ROUND_UP(parent_rate, div);
}

const struct clk_ops krait_div2_clk_ops = {
	.round_rate = krait_div2_round_rate,
	.set_rate = krait_div2_set_rate,
	.recalc_rate = krait_div2_recalc_rate,
};
EXPORT_SYMBOL_GPL(krait_div2_clk_ops);

clk-krait.h

/* SPDX-License-Identifier: GPL-2.0 */

#ifndef __QCOM_CLK_KRAIT_H
#define __QCOM_CLK_KRAIT_H

#include <linux/clk-provider.h>

struct krait_mux_clk {
	unsigned int	*parent_map;
	u32		offset;
	u32		mask;
	u32		shift;
	u32		en_mask;
	bool		lpl;
	u8		safe_sel;
	u8		old_index;
	bool		reparent;
	bool		disable_sec_src_gating;
	unsigned long	safe_rate;

	struct clk_hw	hw;
	struct notifier_block   clk_nb;
};

#define to_krait_mux_clk(_hw) container_of(_hw, struct krait_mux_clk, hw)

extern const struct clk_ops krait_mux_clk_ops;

struct krait_div2_clk {
	u32		offset;
	u32		mask;
	u8		divisor;
	u32		shift;
	bool		lpl;

	struct clk_hw	hw;
};

#define to_krait_div2_clk(_hw) container_of(_hw, struct krait_div2_clk, hw)

extern const struct clk_ops krait_div2_clk_ops;

#endif

krait-cc.c

// SPDX-License-Identifier: GPL-2.0
// Copyright (c) 2018, The Linux Foundation. All rights reserved.

#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/platform_device.h>
#include <linux/err.h>
#include <linux/io.h>
#include <linux/of.h>
#include <linux/of_device.h>
#include <linux/clk.h>
#include <linux/clk-provider.h>
#include <linux/slab.h>

#include "clk-krait.h"

#define	MUX_DIV_2	0x0 /* no bits set divisor set to 2 */

#define QSB_RATE	225000000
#define AUX_RATE	384000000
#define HFPLL_RATE	600000000

static unsigned int sec_mux_map[] = {
	2,
	0,
};

static unsigned int pri_mux_map[] = {
	1,
	2,
	0,
};

/*
 * Notifier function for switching the muxes to safe parent
 * while the hfpll is getting reprogrammed.
 */
static int krait_notifier_cb(struct notifier_block *nb,
			     unsigned long event,
			     void *data)
{
	int ret = 0;
	struct clk_notifier_data *cnd = data;
	struct krait_mux_clk *mux = container_of(nb, struct krait_mux_clk,
						 clk_nb);
	/* Switch to safe parent */
	if (event == PRE_RATE_CHANGE) {
		if (likely(cnd->old_rate > mux->safe_rate) && cnd->new_rate == mux->safe_rate || cnd->old_rate == mux->safe_rate) {
			pr_info("SKIP PRE RATE CHANGE old rate %d new rate %d\n", cnd->old_rate, cnd->new_rate);
			goto exit;
		}

		pr_info("PRE RATE CHANGE old rate %d new rate %d\n", cnd->old_rate, cnd->new_rate);
		mux->old_index = krait_mux_clk_ops.get_parent(&mux->hw);
		ret = krait_mux_clk_ops.set_parent(&mux->hw, mux->safe_sel);
		mux->reparent = false;
	/*
	 * By the time POST_RATE_CHANGE notifier is called,
	 * clk framework itself would have changed the parent for the new rate.
	 * Only otherwise, put back to the old parent.
	 */
	} else if (event == POST_RATE_CHANGE) {
		if (!mux->reparent)
			ret = krait_mux_clk_ops.set_parent(&mux->hw,
							   mux->old_index);
	}

exit:
	return notifier_from_errno(ret);
}

static int krait_notifier_register(struct device *dev, struct clk *clk,
				   struct krait_mux_clk *mux)
{
	int ret = 0;

	mux->clk_nb.notifier_call = krait_notifier_cb;
	ret = devm_clk_notifier_register(dev, clk, &mux->clk_nb);
	if (ret)
		dev_err(dev, "failed to register clock notifier: %d\n", ret);

	return ret;
}

static struct clk *
krait_add_div(struct device *dev, int id, const char *s, unsigned int offset)
{
	struct krait_div2_clk *div;
	static struct clk_parent_data p_data[1];
	struct clk_init_data init = {
		.num_parents = ARRAY_SIZE(p_data),
		.ops = &krait_div2_clk_ops,
		.flags = CLK_SET_RATE_PARENT,
	};
	struct clk *clk;
	char *parent_name;

	div = devm_kzalloc(dev, sizeof(*div), GFP_KERNEL);
	if (!div)
		return ERR_PTR(-ENOMEM);

	div->mask = 0x3;
	div->divisor = MUX_DIV_2;
	div->shift = 6;
	div->lpl = id >= 0;
	div->offset = offset;
	div->hw.init = &init;

	init.name = kasprintf(GFP_KERNEL, "hfpll%s_div", s);
	if (!init.name)
		return ERR_PTR(-ENOMEM);

	init.parent_data = p_data;
	parent_name = kasprintf(GFP_KERNEL, "hfpll%s", s);
	if (!parent_name) {
		clk = ERR_PTR(-ENOMEM);
		goto err_parent_name;
	}

	p_data[0].fw_name = parent_name;
	p_data[0].name = parent_name;

	clk = devm_clk_register(dev, &div->hw);

	kfree(parent_name);
err_parent_name:
	kfree(init.name);

	return clk;
}

static struct clk *
krait_add_sec_mux(struct device *dev, int id, const char *s,
		  unsigned int offset, bool unique_aux)
{
	int ret, cpu;
	struct krait_mux_clk *mux;
	static struct clk_parent_data sec_mux_list[2] = {
		{ .name = "qsb", .fw_name = "qsb" },
		{},
	};
	struct clk_init_data init = {
		.parent_data = sec_mux_list,
		.num_parents = ARRAY_SIZE(sec_mux_list),
		.ops = &krait_mux_clk_ops,
		.flags = CLK_SET_RATE_PARENT,
	};
	struct clk *clk;
	char *parent_name;

	mux = devm_kzalloc(dev, sizeof(*mux), GFP_KERNEL);
	if (!mux)
		return ERR_PTR(-ENOMEM);

	mux->offset = offset;
	mux->lpl = id >= 0;
	mux->mask = 0x3;
	mux->shift = 2;
	mux->parent_map = sec_mux_map;
	mux->hw.init = &init;
	mux->safe_sel = 0;
	mux->safe_rate = QSB_RATE;

	/* Checking for qcom,krait-cc-v1 or qcom,krait-cc-v2 is not
	 * enough to limit this to apq/ipq8064. Directly check machine
	 * compatible to correctly handle this errata.
	 */
	if (of_machine_is_compatible("qcom,ipq8064") ||
	    of_machine_is_compatible("qcom,apq8064"))
		mux->disable_sec_src_gating = true;

	init.name = kasprintf(GFP_KERNEL, "krait%s_sec_mux", s);
	if (!init.name)
		return ERR_PTR(-ENOMEM);

	if (unique_aux) {
		parent_name = kasprintf(GFP_KERNEL, "acpu%s_aux", s);
		if (!parent_name) {
			clk = ERR_PTR(-ENOMEM);
			goto err_aux;
		}
		sec_mux_list[1].fw_name = parent_name;
		sec_mux_list[1].name = parent_name;
	} else {
		sec_mux_list[1].name = "apu_aux";
	}

	clk = devm_clk_register(dev, &mux->hw);
	if (IS_ERR(clk))
		goto err_clk;

	ret = krait_notifier_register(dev, clk, mux);
	if (ret)
		clk = ERR_PTR(ret);

	/* The secondary mux MUST be enabled or clk-krait silently
	 * ignore any request.
	 * Increase refcount for every CPU if it's the L2 secondary mux.
	 */
	if (id < 0)
		for_each_possible_cpu(cpu)
			clk_prepare_enable(clk);
	else
		clk_prepare_enable(clk);

err_clk:
	if (unique_aux)
		kfree(parent_name);
err_aux:
	kfree(init.name);
	return clk;
}

static struct clk *
krait_add_pri_mux(struct device *dev, struct clk *hfpll_div, struct clk *sec_mux,
		  int id, const char *s, unsigned int offset)
{
	int ret;
	struct krait_mux_clk *mux;
	static struct clk_parent_data p_data[3];
	struct clk_init_data init = {
		.parent_data = p_data,
		.num_parents = ARRAY_SIZE(p_data),
		.ops = &krait_mux_clk_ops,
		.flags = CLK_SET_RATE_PARENT,
	};
	struct clk *clk;
	char *hfpll_name;

	mux = devm_kzalloc(dev, sizeof(*mux), GFP_KERNEL);
	if (!mux)
		return ERR_PTR(-ENOMEM);

	mux->mask = 0x3;
	mux->shift = 0;
	mux->offset = offset;
	mux->lpl = id >= 0;
	mux->parent_map = pri_mux_map;
	mux->hw.init = &init;
	mux->safe_sel = 2;
	mux->safe_rate = AUX_RATE;

	init.name = kasprintf(GFP_KERNEL, "krait%s_pri_mux", s);
	if (!init.name)
		return ERR_PTR(-ENOMEM);

	hfpll_name = kasprintf(GFP_KERNEL, "hfpll%s", s);
	if (!hfpll_name) {
		clk = ERR_PTR(-ENOMEM);
		goto err_hfpll;
	}

	p_data[0].fw_name = hfpll_name;
	p_data[0].name = hfpll_name;

	p_data[1].hw = __clk_get_hw(hfpll_div);
	p_data[2].hw = __clk_get_hw(sec_mux);

	clk = devm_clk_register(dev, &mux->hw);
	if (IS_ERR(clk))
		goto err_clk;

	ret = krait_notifier_register(dev, clk, mux);
	if (ret)
		clk = ERR_PTR(ret);

err_clk:
	kfree(hfpll_name);
err_hfpll:
	kfree(init.name);
	return clk;
}

/* id < 0 for L2, otherwise id == physical CPU number */
static struct clk *krait_add_clks(struct device *dev, int id, bool unique_aux)
{
	unsigned int offset;
	void *p = NULL;
	const char *s;
	struct clk *hfpll_div, *sec_mux, *clk;

	if (id >= 0) {
		offset = 0x4501 + (0x1000 * id);
		s = p = kasprintf(GFP_KERNEL, "%d", id);
		if (!s)
			return ERR_PTR(-ENOMEM);
	} else {
		offset = 0x500;
		s = "_l2";
	}

	hfpll_div = krait_add_div(dev, id, s, offset);
	if (IS_ERR(hfpll_div)) {
		clk = hfpll_div;
		goto err;
	}

	sec_mux = krait_add_sec_mux(dev, id, s, offset, unique_aux);
	if (IS_ERR(sec_mux)) {
		clk = sec_mux;
		goto err;
	}

	clk = krait_add_pri_mux(dev, hfpll_div, sec_mux, id, s, offset);
err:
	kfree(p);
	return clk;
}

static struct clk *krait_of_get(struct of_phandle_args *clkspec, void *data)
{
	unsigned int idx = clkspec->args[0];
	struct clk **clks = data;

	if (idx >= 5) {
		pr_err("%s: invalid clock index %d\n", __func__, idx);
		return ERR_PTR(-EINVAL);
	}

	return clks[idx] ? : ERR_PTR(-ENODEV);
}

static const struct of_device_id krait_cc_match_table[] = {
	{ .compatible = "qcom,krait-cc-v1", (void *)1UL },
	{ .compatible = "qcom,krait-cc-v2" },
	{}
};
MODULE_DEVICE_TABLE(of, krait_cc_match_table);

static int krait_cc_probe(struct platform_device *pdev)
{
	struct device *dev = &pdev->dev;
	const struct of_device_id *id;
	unsigned long cur_rate, qsb_rate, pxo_rate;
	int cpu;
	struct clk *clk;
	struct clk **clks;
	struct clk *l2_pri_mux_clk;

	id = of_match_device(krait_cc_match_table, dev);
	if (!id)
		return -ENODEV;

	clk = clk_get(dev, "pxo");
	if (IS_ERR(clk))
		clk = __clk_lookup("pxo_board");

	if (IS_ERR_OR_NULL(clk))
		return clk == NULL ? -ENODEV : PTR_ERR(clk);

	pxo_rate = clk_get_rate(clk);

	/*
	 * Per Documentation qsb should be provided from DTS.
	 * To address old implementation, register the fixed clock anyway.
	 * Rate is 1 because 0 causes problems for __clk_mux_determine_rate
	 */
	clk = clk_get(dev, "qsb");
	if (IS_ERR(clk))
		clk = clk_register_fixed_rate(dev, "qsb", NULL, 0, QSB_RATE);
	if (IS_ERR(clk))
		return PTR_ERR(clk);

	qsb_rate = clk_get_rate(clk);

	if (!id->data) {
		clk = clk_register_fixed_factor(dev, "acpu_aux",
						"gpll0_vote", 0, 1, 2);
		if (IS_ERR(clk))
			return PTR_ERR(clk);
	}

	/* Krait configurations have at most 4 CPUs and one L2 */
	clks = devm_kcalloc(dev, 5, sizeof(*clks), GFP_KERNEL);
	if (!clks)
		return -ENOMEM;

	for_each_possible_cpu(cpu) {
		clk = krait_add_clks(dev, cpu, id->data);
		if (IS_ERR(clk))
			return PTR_ERR(clk);
		clks[cpu] = clk;
	}

	l2_pri_mux_clk = krait_add_clks(dev, -1, id->data);
	if (IS_ERR(l2_pri_mux_clk))
		return PTR_ERR(l2_pri_mux_clk);
	clks[4] = l2_pri_mux_clk;

	/*
	 * We don't want the CPU or L2 clocks to be turned off at late init
	 * if CPUFREQ or HOTPLUG configs are disabled. So, bump up the
	 * refcount of these clocks. Any cpufreq/hotplug manager can assume
	 * that the clocks have already been prepared and enabled by the time
	 * they take over.
	 */
	for_each_online_cpu(cpu) {
		clk_prepare_enable(l2_pri_mux_clk);
		WARN(clk_prepare_enable(clks[cpu]),
		     "Unable to turn on CPU%d clock", cpu);
	}

	/*
	 * Force reinit of HFPLLs and muxes to overwrite any potential
	 * incorrect configuration of HFPLLs and muxes by the bootloader.
	 * While at it, also make sure the cores are running at known rates
	 * and print the current rate.
	 *
	 * The clocks are set to aux clock rate first to make sure the
	 * secondary mux is not sourcing off of QSB. The rate is then set to
	 * two different rates to force a HFPLL reinit under all
	 * circumstances.
	 */
	for (cpu = 0; cpu < 5; cpu++) {
		const char *l2_s = "L2";
		char cpu_s[5];

		clk = clks[cpu];
		if (!clk)
			continue;

		if (cpu < 4)
			snprintf(cpu_s, 5, "CPU%d", cpu);

		cur_rate = clk_get_rate(clk);
		if (cur_rate == qsb_rate || cur_rate == pxo_rate) {
			dev_info(dev, "%s @ %s rate. Forcing new rate.\n",
				 cpu < 4 ? cpu_s : l2_s,
				 cur_rate == qsb_rate ? "QSB" : "PXO");
			cur_rate = AUX_RATE;
		}

		clk_set_rate(clk, AUX_RATE);
		clk_set_rate(clk, HFPLL_RATE);
		clk_set_rate(clk, cur_rate);
		dev_info(dev, "%s @ %lu KHz\n", cpu < 4 ? cpu_s : l2_s,
			 clk_get_rate(clk) / 1000);
	}

	of_clk_add_provider(dev->of_node, krait_of_get, clks);

	return 0;
}

static struct platform_driver krait_cc_driver = {
	.probe = krait_cc_probe,
	.driver = {
		.name = "krait-cc",
		.of_match_table = krait_cc_match_table,
	},
};
module_platform_driver(krait_cc_driver);

MODULE_DESCRIPTION("Krait CPU Clock Driver");
MODULE_LICENSE("GPL v2");
MODULE_ALIAS("platform:krait-cc");