vz.c

		if (!cpu_guest_has_badinstr)
			return -EINVAL;
		*v = read_gc0_badinstr();
		break;
	case KVM_REG_MIPS_CP0_BADINSTRP:
		if (!cpu_guest_has_badinstrp)
			return -EINVAL;
		*v = read_gc0_badinstrp();
		break;
	case KVM_REG_MIPS_CP0_COUNT:
		*v = kvm_mips_read_count(vcpu);
		break;
	case KVM_REG_MIPS_CP0_ENTRYHI:
		*v = (long)read_gc0_entryhi();
		break;
	case KVM_REG_MIPS_CP0_COMPARE:
		*v = (long)read_gc0_compare();
		break;
	case KVM_REG_MIPS_CP0_STATUS:
		*v = (long)read_gc0_status();
		break;
	case KVM_REG_MIPS_CP0_INTCTL:
		*v = read_gc0_intctl();
		break;
	case KVM_REG_MIPS_CP0_CAUSE:
		*v = (long)read_gc0_cause();
		break;
	case KVM_REG_MIPS_CP0_EPC:
		*v = (long)read_gc0_epc();
		break;
	case KVM_REG_MIPS_CP0_PRID:
		switch (boot_cpu_type()) {
		case CPU_CAVIUM_OCTEON3:
			/* Octeon III has a read-only guest.PRid */
			*v = read_gc0_prid();
			break;
		default:
			*v = (long)kvm_read_c0_guest_prid(cop0);
			break;
		}
		break;
	case KVM_REG_MIPS_CP0_EBASE:
		*v = kvm_vz_read_gc0_ebase();
		break;
	case KVM_REG_MIPS_CP0_CONFIG:
		*v = read_gc0_config();
		break;
	case KVM_REG_MIPS_CP0_CONFIG1:
		if (!cpu_guest_has_conf1)
			return -EINVAL;
		*v = read_gc0_config1();
		break;
	case KVM_REG_MIPS_CP0_CONFIG2:
		if (!cpu_guest_has_conf2)
			return -EINVAL;
		*v = read_gc0_config2();
		break;
	case KVM_REG_MIPS_CP0_CONFIG3:
		if (!cpu_guest_has_conf3)
			return -EINVAL;
		*v = read_gc0_config3();
		break;
	case KVM_REG_MIPS_CP0_CONFIG4:
		if (!cpu_guest_has_conf4)
			return -EINVAL;
		*v = read_gc0_config4();
		break;
	case KVM_REG_MIPS_CP0_CONFIG5:
		if (!cpu_guest_has_conf5)
			return -EINVAL;
		*v = read_gc0_config5();
		break;
	case KVM_REG_MIPS_CP0_CONFIG6:
		*v = kvm_read_sw_gc0_config6(cop0);
		break;
	case KVM_REG_MIPS_CP0_MAAR(0) ... KVM_REG_MIPS_CP0_MAAR(0x3f):
		if (!cpu_guest_has_maar || cpu_guest_has_dyn_maar)
			return -EINVAL;
		idx = reg->id - KVM_REG_MIPS_CP0_MAAR(0);
		if (idx >= ARRAY_SIZE(vcpu->arch.maar))
			return -EINVAL;
		*v = vcpu->arch.maar[idx];
		break;
	case KVM_REG_MIPS_CP0_MAARI:
		if (!cpu_guest_has_maar || cpu_guest_has_dyn_maar)
			return -EINVAL;
		*v = kvm_read_sw_gc0_maari(vcpu->arch.cop0);
		break;
#ifdef CONFIG_64BIT
	case KVM_REG_MIPS_CP0_XCONTEXT:
		*v = read_gc0_xcontext();
		break;
#endif
	case KVM_REG_MIPS_CP0_ERROREPC:
		*v = (long)read_gc0_errorepc();
		break;
	case KVM_REG_MIPS_CP0_KSCRATCH1 ... KVM_REG_MIPS_CP0_KSCRATCH6:
		idx = reg->id - KVM_REG_MIPS_CP0_KSCRATCH1 + 2;
		if (!cpu_guest_has_kscr(idx))
			return -EINVAL;
		switch (idx) {
		case 2:
			*v = (long)read_gc0_kscratch1();
			break;
		case 3:
			*v = (long)read_gc0_kscratch2();
			break;
		case 4:
			*v = (long)read_gc0_kscratch3();
			break;
		case 5:
			*v = (long)read_gc0_kscratch4();
			break;
		case 6:
			*v = (long)read_gc0_kscratch5();
			break;
		case 7:
			*v = (long)read_gc0_kscratch6();
			break;
		}
		break;
	case KVM_REG_MIPS_COUNT_CTL:
		*v = vcpu->arch.count_ctl;
		break;
	case KVM_REG_MIPS_COUNT_RESUME:
		*v = ktime_to_ns(vcpu->arch.count_resume);
		break;
	case KVM_REG_MIPS_COUNT_HZ:
		*v = vcpu->arch.count_hz;
		break;
	default:
		return -EINVAL;
	}
	return 0;
}

static int kvm_vz_set_one_reg(struct kvm_vcpu *vcpu,
			      const struct kvm_one_reg *reg,
			      s64 v)
{
	struct mips_coproc *cop0 = vcpu->arch.cop0;
	unsigned int idx;
	int ret = 0;
	unsigned int cur, change;

	switch (reg->id) {
	case KVM_REG_MIPS_CP0_INDEX:
		write_gc0_index(v);
		break;
	case KVM_REG_MIPS_CP0_ENTRYLO0:
		write_gc0_entrylo0(entrylo_user_to_kvm(v));
		break;
	case KVM_REG_MIPS_CP0_ENTRYLO1:
		write_gc0_entrylo1(entrylo_user_to_kvm(v));
		break;
	case KVM_REG_MIPS_CP0_CONTEXT:
		write_gc0_context(v);
		break;
	case KVM_REG_MIPS_CP0_CONTEXTCONFIG:
		if (!cpu_guest_has_contextconfig)
			return -EINVAL;
		write_gc0_contextconfig(v);
		break;
	case KVM_REG_MIPS_CP0_USERLOCAL:
		if (!cpu_guest_has_userlocal)
			return -EINVAL;
		write_gc0_userlocal(v);
		break;
#ifdef CONFIG_64BIT
	case KVM_REG_MIPS_CP0_XCONTEXTCONFIG:
		if (!cpu_guest_has_contextconfig)
			return -EINVAL;
		write_gc0_xcontextconfig(v);
		break;
#endif
	case KVM_REG_MIPS_CP0_PAGEMASK:
		write_gc0_pagemask(v);
		break;
	case KVM_REG_MIPS_CP0_PAGEGRAIN:
		write_gc0_pagegrain(v);
		break;
	case KVM_REG_MIPS_CP0_SEGCTL0:
		if (!cpu_guest_has_segments)
			return -EINVAL;
		write_gc0_segctl0(v);
		break;
	case KVM_REG_MIPS_CP0_SEGCTL1:
		if (!cpu_guest_has_segments)
			return -EINVAL;
		write_gc0_segctl1(v);
		break;
	case KVM_REG_MIPS_CP0_SEGCTL2:
		if (!cpu_guest_has_segments)
			return -EINVAL;
		write_gc0_segctl2(v);
		break;
	case KVM_REG_MIPS_CP0_PWBASE:
		if (!cpu_guest_has_htw && !cpu_guest_has_ldpte)
			return -EINVAL;
		write_gc0_pwbase(v);
		break;
	case KVM_REG_MIPS_CP0_PWFIELD:
		if (!cpu_guest_has_htw && !cpu_guest_has_ldpte)
			return -EINVAL;
		write_gc0_pwfield(v);
		break;
	case KVM_REG_MIPS_CP0_PWSIZE:
		if (!cpu_guest_has_htw && !cpu_guest_has_ldpte)
			return -EINVAL;
		write_gc0_pwsize(v);
		break;
	case KVM_REG_MIPS_CP0_WIRED:
		change_gc0_wired(MIPSR6_WIRED_WIRED, v);
		break;
	case KVM_REG_MIPS_CP0_PWCTL:
		if (!cpu_guest_has_htw && !cpu_guest_has_ldpte)
			return -EINVAL;
		write_gc0_pwctl(v);
		break;
	case KVM_REG_MIPS_CP0_HWRENA:
		write_gc0_hwrena(v);
		break;
	case KVM_REG_MIPS_CP0_BADVADDR:
		write_gc0_badvaddr(v);
		break;
	case KVM_REG_MIPS_CP0_BADINSTR:
		if (!cpu_guest_has_badinstr)
			return -EINVAL;
		write_gc0_badinstr(v);
		break;
	case KVM_REG_MIPS_CP0_BADINSTRP:
		if (!cpu_guest_has_badinstrp)
			return -EINVAL;
		write_gc0_badinstrp(v);
		break;
	case KVM_REG_MIPS_CP0_COUNT:
		kvm_mips_write_count(vcpu, v);
		break;
	case KVM_REG_MIPS_CP0_ENTRYHI:
		write_gc0_entryhi(v);
		break;
	case KVM_REG_MIPS_CP0_COMPARE:
		kvm_mips_write_compare(vcpu, v, false);
		break;
	case KVM_REG_MIPS_CP0_STATUS:
		write_gc0_status(v);
		break;
	case KVM_REG_MIPS_CP0_INTCTL:
		write_gc0_intctl(v);
		break;
	case KVM_REG_MIPS_CP0_CAUSE:
		/*
		 * If the timer is stopped or started (DC bit) it must look
		 * atomic with changes to the timer interrupt pending bit (TI).
		 * A timer interrupt should not happen in between.
		 */
		if ((read_gc0_cause() ^ v) & CAUSEF_DC) {
			if (v & CAUSEF_DC) {
				/* disable timer first */
				kvm_mips_count_disable_cause(vcpu);
				change_gc0_cause((u32)~CAUSEF_DC, v);
			} else {
				/* enable timer last */
				change_gc0_cause((u32)~CAUSEF_DC, v);
				kvm_mips_count_enable_cause(vcpu);
			}
		} else {
			write_gc0_cause(v);
		}
		break;
	case KVM_REG_MIPS_CP0_EPC:
		write_gc0_epc(v);
		break;
	case KVM_REG_MIPS_CP0_PRID:
		switch (boot_cpu_type()) {
		case CPU_CAVIUM_OCTEON3:
			/* Octeon III has a guest.PRid, but its read-only */
			break;
		default:
			kvm_write_c0_guest_prid(cop0, v);
			break;
		}
		break;
	case KVM_REG_MIPS_CP0_EBASE:
		kvm_vz_write_gc0_ebase(v);
		break;
	case KVM_REG_MIPS_CP0_CONFIG:
		cur = read_gc0_config();
		change = (cur ^ v) & kvm_vz_config_user_wrmask(vcpu);
		if (change) {
			v = cur ^ change;
			write_gc0_config(v);
		}
		break;
	case KVM_REG_MIPS_CP0_CONFIG1:
		if (!cpu_guest_has_conf1)
			break;
		cur = read_gc0_config1();
		change = (cur ^ v) & kvm_vz_config1_user_wrmask(vcpu);
		if (change) {
			v = cur ^ change;
			write_gc0_config1(v);
		}
		break;
	case KVM_REG_MIPS_CP0_CONFIG2:
		if (!cpu_guest_has_conf2)
			break;
		cur = read_gc0_config2();
		change = (cur ^ v) & kvm_vz_config2_user_wrmask(vcpu);
		if (change) {
			v = cur ^ change;
			write_gc0_config2(v);
		}
		break;
	case KVM_REG_MIPS_CP0_CONFIG3:
		if (!cpu_guest_has_conf3)
			break;
		cur = read_gc0_config3();
		change = (cur ^ v) & kvm_vz_config3_user_wrmask(vcpu);
		if (change) {
			v = cur ^ change;
			write_gc0_config3(v);
		}
		break;
	case KVM_REG_MIPS_CP0_CONFIG4:
		if (!cpu_guest_has_conf4)
			break;
		cur = read_gc0_config4();
		change = (cur ^ v) & kvm_vz_config4_user_wrmask(vcpu);
		if (change) {
			v = cur ^ change;
			write_gc0_config4(v);
		}
		break;
	case KVM_REG_MIPS_CP0_CONFIG5:
		if (!cpu_guest_has_conf5)
			break;
		cur = read_gc0_config5();
		change = (cur ^ v) & kvm_vz_config5_user_wrmask(vcpu);
		if (change) {
			v = cur ^ change;
			write_gc0_config5(v);
		}
		break;
	case KVM_REG_MIPS_CP0_CONFIG6:
		cur = kvm_read_sw_gc0_config6(cop0);
		change = (cur ^ v) & kvm_vz_config6_user_wrmask(vcpu);
		if (change) {
			v = cur ^ change;
			kvm_write_sw_gc0_config6(cop0, (int)v);
		}
		break;
	case KVM_REG_MIPS_CP0_MAAR(0) ... KVM_REG_MIPS_CP0_MAAR(0x3f):
		if (!cpu_guest_has_maar || cpu_guest_has_dyn_maar)
			return -EINVAL;
		idx = reg->id - KVM_REG_MIPS_CP0_MAAR(0);
		if (idx >= ARRAY_SIZE(vcpu->arch.maar))
			return -EINVAL;
		vcpu->arch.maar[idx] = mips_process_maar(dmtc_op, v);
		break;
	case KVM_REG_MIPS_CP0_MAARI:
		if (!cpu_guest_has_maar || cpu_guest_has_dyn_maar)
			return -EINVAL;
		kvm_write_maari(vcpu, v);
		break;
#ifdef CONFIG_64BIT
	case KVM_REG_MIPS_CP0_XCONTEXT:
		write_gc0_xcontext(v);
		break;
#endif
	case KVM_REG_MIPS_CP0_ERROREPC:
		write_gc0_errorepc(v);
		break;
	case KVM_REG_MIPS_CP0_KSCRATCH1 ... KVM_REG_MIPS_CP0_KSCRATCH6:
		idx = reg->id - KVM_REG_MIPS_CP0_KSCRATCH1 + 2;
		if (!cpu_guest_has_kscr(idx))
			return -EINVAL;
		switch (idx) {
		case 2:
			write_gc0_kscratch1(v);
			break;
		case 3:
			write_gc0_kscratch2(v);
			break;
		case 4:
			write_gc0_kscratch3(v);
			break;
		case 5:
			write_gc0_kscratch4(v);
			break;
		case 6:
			write_gc0_kscratch5(v);
			break;
		case 7:
			write_gc0_kscratch6(v);
			break;
		}
		break;
	case KVM_REG_MIPS_COUNT_CTL:
		ret = kvm_mips_set_count_ctl(vcpu, v);
		break;
	case KVM_REG_MIPS_COUNT_RESUME:
		ret = kvm_mips_set_count_resume(vcpu, v);
		break;
	case KVM_REG_MIPS_COUNT_HZ:
		ret = kvm_mips_set_count_hz(vcpu, v);
		break;
	default:
		return -EINVAL;
	}
	return ret;
}

#define guestid_cache(cpu)	(cpu_data[cpu].guestid_cache)
static void kvm_vz_get_new_guestid(unsigned long cpu, struct kvm_vcpu *vcpu)
{
	unsigned long guestid = guestid_cache(cpu);

	if (!(++guestid & GUESTID_MASK)) {
		if (cpu_has_vtag_icache)
			flush_icache_all();

		if (!guestid)		/* fix version if needed */
			guestid = GUESTID_FIRST_VERSION;

		++guestid;		/* guestid 0 reserved for root */

		/* start new guestid cycle */
		kvm_vz_local_flush_roottlb_all_guests();
		kvm_vz_local_flush_guesttlb_all();
	}

	guestid_cache(cpu) = guestid;
}

/* Returns 1 if the guest TLB may be clobbered */
static int kvm_vz_check_requests(struct kvm_vcpu *vcpu, int cpu)
{
	int ret = 0;
	int i;

	if (!kvm_request_pending(vcpu))
		return 0;

	if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
		if (cpu_has_guestid) {
			/* Drop all GuestIDs for this VCPU */
			for_each_possible_cpu(i)
				vcpu->arch.vzguestid[i] = 0;
			/* This will clobber guest TLB contents too */
			ret = 1;
		}
		/*
		 * For Root ASID Dealias (RAD) we don't do anything here, but we
		 * still need the request to ensure we recheck asid_flush_mask.
		 * We can still return 0 as only the root TLB will be affected
		 * by a root ASID flush.
		 */
	}

	return ret;
}

static void kvm_vz_vcpu_save_wired(struct kvm_vcpu *vcpu)
{
	unsigned int wired = read_gc0_wired();
	struct kvm_mips_tlb *tlbs;
	int i;

	/* Expand the wired TLB array if necessary */
	wired &= MIPSR6_WIRED_WIRED;
	if (wired > vcpu->arch.wired_tlb_limit) {
		tlbs = krealloc(vcpu->arch.wired_tlb, wired *
				sizeof(*vcpu->arch.wired_tlb), GFP_ATOMIC);
		if (WARN_ON(!tlbs)) {
			/* Save whatever we can */
			wired = vcpu->arch.wired_tlb_limit;
		} else {
			vcpu->arch.wired_tlb = tlbs;
			vcpu->arch.wired_tlb_limit = wired;
		}
	}

	if (wired)
		/* Save wired entries from the guest TLB */
		kvm_vz_save_guesttlb(vcpu->arch.wired_tlb, 0, wired);
	/* Invalidate any dropped entries since last time */
	for (i = wired; i < vcpu->arch.wired_tlb_used; ++i) {
		vcpu->arch.wired_tlb[i].tlb_hi = UNIQUE_GUEST_ENTRYHI(i);
		vcpu->arch.wired_tlb[i].tlb_lo[0] = 0;
		vcpu->arch.wired_tlb[i].tlb_lo[1] = 0;
		vcpu->arch.wired_tlb[i].tlb_mask = 0;
	}
	vcpu->arch.wired_tlb_used = wired;
}

static void kvm_vz_vcpu_load_wired(struct kvm_vcpu *vcpu)
{
	/* Load wired entries into the guest TLB */
	if (vcpu->arch.wired_tlb)
		kvm_vz_load_guesttlb(vcpu->arch.wired_tlb, 0,
				     vcpu->arch.wired_tlb_used);
}

static void kvm_vz_vcpu_load_tlb(struct kvm_vcpu *vcpu, int cpu)
{
	struct kvm *kvm = vcpu->kvm;
	struct mm_struct *gpa_mm = &kvm->arch.gpa_mm;
	bool migrated;

	/*
	 * Are we entering guest context on a different CPU to last time?
	 * If so, the VCPU's guest TLB state on this CPU may be stale.
	 */
	migrated = (vcpu->arch.last_exec_cpu != cpu);
	vcpu->arch.last_exec_cpu = cpu;

	/*
	 * A vcpu's GuestID is set in GuestCtl1.ID when the vcpu is loaded and
	 * remains set until another vcpu is loaded in.  As a rule GuestRID
	 * remains zeroed when in root context unless the kernel is busy
	 * manipulating guest tlb entries.
	 */
	if (cpu_has_guestid) {
		/*
		 * Check if our GuestID is of an older version and thus invalid.
		 *
		 * We also discard the stored GuestID if we've executed on
		 * another CPU, as the guest mappings may have changed without
		 * hypervisor knowledge.
		 */
		if (migrated ||
		    (vcpu->arch.vzguestid[cpu] ^ guestid_cache(cpu)) &
					GUESTID_VERSION_MASK) {
			kvm_vz_get_new_guestid(cpu, vcpu);
			vcpu->arch.vzguestid[cpu] = guestid_cache(cpu);
			trace_kvm_guestid_change(vcpu,
						 vcpu->arch.vzguestid[cpu]);
		}

		/* Restore GuestID */
		change_c0_guestctl1(GUESTID_MASK, vcpu->arch.vzguestid[cpu]);
	} else {
		/*
		 * The Guest TLB only stores a single guest's TLB state, so
		 * flush it if another VCPU has executed on this CPU.
		 *
		 * We also flush if we've executed on another CPU, as the guest
		 * mappings may have changed without hypervisor knowledge.
		 */
		if (migrated || last_exec_vcpu[cpu] != vcpu)
			kvm_vz_local_flush_guesttlb_all();
		last_exec_vcpu[cpu] = vcpu;

		/*
		 * Root ASID dealiases guest GPA mappings in the root TLB.
		 * Allocate new root ASID if needed.
		 */
		if (cpumask_test_and_clear_cpu(cpu, &kvm->arch.asid_flush_mask))
			get_new_mmu_context(gpa_mm);
		else
			check_mmu_context(gpa_mm);
	}
}

static int kvm_vz_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
{
	struct mips_coproc *cop0 = vcpu->arch.cop0;
	bool migrated, all;

	/*
	 * Have we migrated to a different CPU?
	 * If so, any old guest TLB state may be stale.
	 */
	migrated = (vcpu->arch.last_sched_cpu != cpu);

	/*
	 * Was this the last VCPU to run on this CPU?
	 * If not, any old guest state from this VCPU will have been clobbered.
	 */
	all = migrated || (last_vcpu[cpu] != vcpu);
	last_vcpu[cpu] = vcpu;

	/*
	 * Restore CP0_Wired unconditionally as we clear it after use, and
	 * restore wired guest TLB entries (while in guest context).
	 */
	kvm_restore_gc0_wired(cop0);
	if (current->flags & PF_VCPU) {
		tlbw_use_hazard();
		kvm_vz_vcpu_load_tlb(vcpu, cpu);
		kvm_vz_vcpu_load_wired(vcpu);
	}

	/*
	 * Restore timer state regardless, as e.g. Cause.TI can change over time
	 * if left unmaintained.
	 */
	kvm_vz_restore_timer(vcpu);

	/* Set MC bit if we want to trace guest mode changes */
	if (kvm_trace_guest_mode_change)
		set_c0_guestctl0(MIPS_GCTL0_MC);
	else
		clear_c0_guestctl0(MIPS_GCTL0_MC);

	/* Don't bother restoring registers multiple times unless necessary */
	if (!all)
		return 0;

	/*
	 * Restore config registers first, as some implementations restrict
	 * writes to other registers when the corresponding feature bits aren't
	 * set. For example Status.CU1 cannot be set unless Config1.FP is set.
	 */
	kvm_restore_gc0_config(cop0);
	if (cpu_guest_has_conf1)
		kvm_restore_gc0_config1(cop0);
	if (cpu_guest_has_conf2)
		kvm_restore_gc0_config2(cop0);
	if (cpu_guest_has_conf3)
		kvm_restore_gc0_config3(cop0);
	if (cpu_guest_has_conf4)
		kvm_restore_gc0_config4(cop0);
	if (cpu_guest_has_conf5)
		kvm_restore_gc0_config5(cop0);
	if (cpu_guest_has_conf6)
		kvm_restore_gc0_config6(cop0);
	if (cpu_guest_has_conf7)
		kvm_restore_gc0_config7(cop0);

	kvm_restore_gc0_index(cop0);
	kvm_restore_gc0_entrylo0(cop0);
	kvm_restore_gc0_entrylo1(cop0);
	kvm_restore_gc0_context(cop0);
	if (cpu_guest_has_contextconfig)
		kvm_restore_gc0_contextconfig(cop0);
#ifdef CONFIG_64BIT
	kvm_restore_gc0_xcontext(cop0);
	if (cpu_guest_has_contextconfig)
		kvm_restore_gc0_xcontextconfig(cop0);
#endif
	kvm_restore_gc0_pagemask(cop0);
	kvm_restore_gc0_pagegrain(cop0);
	kvm_restore_gc0_hwrena(cop0);
	kvm_restore_gc0_badvaddr(cop0);
	kvm_restore_gc0_entryhi(cop0);
	kvm_restore_gc0_status(cop0);
	kvm_restore_gc0_intctl(cop0);
	kvm_restore_gc0_epc(cop0);
	kvm_vz_write_gc0_ebase(kvm_read_sw_gc0_ebase(cop0));
	if (cpu_guest_has_userlocal)
		kvm_restore_gc0_userlocal(cop0);

	kvm_restore_gc0_errorepc(cop0);

	/* restore KScratch registers if enabled in guest */
	if (cpu_guest_has_conf4) {
		if (cpu_guest_has_kscr(2))
			kvm_restore_gc0_kscratch1(cop0);
		if (cpu_guest_has_kscr(3))
			kvm_restore_gc0_kscratch2(cop0);
		if (cpu_guest_has_kscr(4))
			kvm_restore_gc0_kscratch3(cop0);
		if (cpu_guest_has_kscr(5))
			kvm_restore_gc0_kscratch4(cop0);
		if (cpu_guest_has_kscr(6))
			kvm_restore_gc0_kscratch5(cop0);
		if (cpu_guest_has_kscr(7))
			kvm_restore_gc0_kscratch6(cop0);
	}

	if (cpu_guest_has_badinstr)
		kvm_restore_gc0_badinstr(cop0);
	if (cpu_guest_has_badinstrp)
		kvm_restore_gc0_badinstrp(cop0);

	if (cpu_guest_has_segments) {
		kvm_restore_gc0_segctl0(cop0);
		kvm_restore_gc0_segctl1(cop0);
		kvm_restore_gc0_segctl2(cop0);
	}

	/* restore HTW registers */
	if (cpu_guest_has_htw || cpu_guest_has_ldpte) {
		kvm_restore_gc0_pwbase(cop0);
		kvm_restore_gc0_pwfield(cop0);
		kvm_restore_gc0_pwsize(cop0);
		kvm_restore_gc0_pwctl(cop0);
	}

	/* restore Root.GuestCtl2 from unused Guest guestctl2 register */
	if (cpu_has_guestctl2)
		write_c0_guestctl2(
			cop0->reg[MIPS_CP0_GUESTCTL2][MIPS_CP0_GUESTCTL2_SEL]);

	/*
	 * We should clear linked load bit to break interrupted atomics. This
	 * prevents a SC on the next VCPU from succeeding by matching a LL on
	 * the previous VCPU.
	 */
	if (vcpu->kvm->created_vcpus > 1)
		write_gc0_lladdr(0);

	return 0;
}

static int kvm_vz_vcpu_put(struct kvm_vcpu *vcpu, int cpu)
{
	struct mips_coproc *cop0 = vcpu->arch.cop0;

	if (current->flags & PF_VCPU)
		kvm_vz_vcpu_save_wired(vcpu);

	kvm_lose_fpu(vcpu);

	kvm_save_gc0_index(cop0);
	kvm_save_gc0_entrylo0(cop0);
	kvm_save_gc0_entrylo1(cop0);
	kvm_save_gc0_context(cop0);
	if (cpu_guest_has_contextconfig)
		kvm_save_gc0_contextconfig(cop0);
#ifdef CONFIG_64BIT
	kvm_save_gc0_xcontext(cop0);
	if (cpu_guest_has_contextconfig)
		kvm_save_gc0_xcontextconfig(cop0);
#endif
	kvm_save_gc0_pagemask(cop0);
	kvm_save_gc0_pagegrain(cop0);
	kvm_save_gc0_wired(cop0);
	/* allow wired TLB entries to be overwritten */
	clear_gc0_wired(MIPSR6_WIRED_WIRED);
	kvm_save_gc0_hwrena(cop0);
	kvm_save_gc0_badvaddr(cop0);
	kvm_save_gc0_entryhi(cop0);
	kvm_save_gc0_status(cop0);
	kvm_save_gc0_intctl(cop0);
	kvm_save_gc0_epc(cop0);
	kvm_write_sw_gc0_ebase(cop0, kvm_vz_read_gc0_ebase());
	if (cpu_guest_has_userlocal)
		kvm_save_gc0_userlocal(cop0);

	/* only save implemented config registers */
	kvm_save_gc0_config(cop0);
	if (cpu_guest_has_conf1)
		kvm_save_gc0_config1(cop0);
	if (cpu_guest_has_conf2)
		kvm_save_gc0_config2(cop0);
	if (cpu_guest_has_conf3)
		kvm_save_gc0_config3(cop0);
	if (cpu_guest_has_conf4)
		kvm_save_gc0_config4(cop0);
	if (cpu_guest_has_conf5)
		kvm_save_gc0_config5(cop0);
	if (cpu_guest_has_conf6)
		kvm_save_gc0_config6(cop0);
	if (cpu_guest_has_conf7)
		kvm_save_gc0_config7(cop0);

	kvm_save_gc0_errorepc(cop0);

	/* save KScratch registers if enabled in guest */
	if (cpu_guest_has_conf4) {
		if (cpu_guest_has_kscr(2))
			kvm_save_gc0_kscratch1(cop0);
		if (cpu_guest_has_kscr(3))
			kvm_save_gc0_kscratch2(cop0);
		if (cpu_guest_has_kscr(4))
			kvm_save_gc0_kscratch3(cop0);
		if (cpu_guest_has_kscr(5))
			kvm_save_gc0_kscratch4(cop0);
		if (cpu_guest_has_kscr(6))
			kvm_save_gc0_kscratch5(cop0);
		if (cpu_guest_has_kscr(7))
			kvm_save_gc0_kscratch6(cop0);
	}

	if (cpu_guest_has_badinstr)
		kvm_save_gc0_badinstr(cop0);
	if (cpu_guest_has_badinstrp)
		kvm_save_gc0_badinstrp(cop0);

	if (cpu_guest_has_segments) {
		kvm_save_gc0_segctl0(cop0);
		kvm_save_gc0_segctl1(cop0);
		kvm_save_gc0_segctl2(cop0);
	}

	/* save HTW registers if enabled in guest */
	if (cpu_guest_has_ldpte || (cpu_guest_has_htw &&
	    kvm_read_sw_gc0_config3(cop0) & MIPS_CONF3_PW)) {
		kvm_save_gc0_pwbase(cop0);
		kvm_save_gc0_pwfield(cop0);
		kvm_save_gc0_pwsize(cop0);
		kvm_save_gc0_pwctl(cop0);
	}

	kvm_vz_save_timer(vcpu);

	/* save Root.GuestCtl2 in unused Guest guestctl2 register */
	if (cpu_has_guestctl2)
		cop0->reg[MIPS_CP0_GUESTCTL2][MIPS_CP0_GUESTCTL2_SEL] =
			read_c0_guestctl2();

	return 0;
}

/**
 * kvm_vz_resize_guest_vtlb() - Attempt to resize guest VTLB.
 * @size:	Number of guest VTLB entries (0 < @size <= root VTLB entries).
 *
 * Attempt to resize the guest VTLB by writing guest Config registers. This is
 * necessary for cores with a shared root/guest TLB to avoid overlap with wired
 * entries in the root VTLB.
 *
 * Returns:	The resulting guest VTLB size.
 */
static unsigned int kvm_vz_resize_guest_vtlb(unsigned int size)
{
	unsigned int config4 = 0, ret = 0, limit;

	/* Write MMUSize - 1 into guest Config registers */
	if (cpu_guest_has_conf1)
		change_gc0_config1(MIPS_CONF1_TLBS,
				   (size - 1) << MIPS_CONF1_TLBS_SHIFT);
	if (cpu_guest_has_conf4) {
		config4 = read_gc0_config4();
		if (cpu_has_mips_r6 || (config4 & MIPS_CONF4_MMUEXTDEF) ==
		    MIPS_CONF4_MMUEXTDEF_VTLBSIZEEXT) {
			config4 &= ~MIPS_CONF4_VTLBSIZEEXT;
			config4 |= ((size - 1) >> MIPS_CONF1_TLBS_SIZE) <<
				MIPS_CONF4_VTLBSIZEEXT_SHIFT;
		} else if ((config4 & MIPS_CONF4_MMUEXTDEF) ==
			   MIPS_CONF4_MMUEXTDEF_MMUSIZEEXT) {
			config4 &= ~MIPS_CONF4_MMUSIZEEXT;
			config4 |= ((size - 1) >> MIPS_CONF1_TLBS_SIZE) <<
				MIPS_CONF4_MMUSIZEEXT_SHIFT;
		}
		write_gc0_config4(config4);
	}

	/*
	 * Set Guest.Wired.Limit = 0 (no limit up to Guest.MMUSize-1), unless it
	 * would exceed Root.Wired.Limit (clearing Guest.Wired.Wired so write
	 * not dropped)
	 */
	if (cpu_has_mips_r6) {
		limit = (read_c0_wired() & MIPSR6_WIRED_LIMIT) >>
						MIPSR6_WIRED_LIMIT_SHIFT;
		if (size - 1 <= limit)
			limit = 0;
		write_gc0_wired(limit << MIPSR6_WIRED_LIMIT_SHIFT);
	}

	/* Read back MMUSize - 1 */
	back_to_back_c0_hazard();
	if (cpu_guest_has_conf1)
		ret = (read_gc0_config1() & MIPS_CONF1_TLBS) >>
						MIPS_CONF1_TLBS_SHIFT;
	if (config4) {
		if (cpu_has_mips_r6 || (config4 & MIPS_CONF4_MMUEXTDEF) ==
		    MIPS_CONF4_MMUEXTDEF_VTLBSIZEEXT)
			ret |= ((config4 & MIPS_CONF4_VTLBSIZEEXT) >>
				MIPS_CONF4_VTLBSIZEEXT_SHIFT) <<
				MIPS_CONF1_TLBS_SIZE;
		else if ((config4 & MIPS_CONF4_MMUEXTDEF) ==
			 MIPS_CONF4_MMUEXTDEF_MMUSIZEEXT)
			ret |= ((config4 & MIPS_CONF4_MMUSIZEEXT) >>
				MIPS_CONF4_MMUSIZEEXT_SHIFT) <<
				MIPS_CONF1_TLBS_SIZE;
	}
	return ret + 1;
}

static int kvm_vz_hardware_enable(void)
{
	unsigned int mmu_size, guest_mmu_size, ftlb_size;
	u64 guest_cvmctl, cvmvmconfig;

	switch (current_cpu_type()) {
	case CPU_CAVIUM_OCTEON3:
		/* Set up guest timer/perfcount IRQ lines */
		guest_cvmctl = read_gc0_cvmctl();
		guest_cvmctl &= ~CVMCTL_IPTI;
		guest_cvmctl |= 7ull << CVMCTL_IPTI_SHIFT;
		guest_cvmctl &= ~CVMCTL_IPPCI;
		guest_cvmctl |= 6ull << CVMCTL_IPPCI_SHIFT;
		write_gc0_cvmctl(guest_cvmctl);

		cvmvmconfig = read_c0_cvmvmconfig();
		/* No I/O hole translation. */
		cvmvmconfig |= CVMVMCONF_DGHT;
		/* Halve the root MMU size */
		mmu_size = ((cvmvmconfig & CVMVMCONF_MMUSIZEM1)
			    >> CVMVMCONF_MMUSIZEM1_S) + 1;
		guest_mmu_size = mmu_size / 2;
		mmu_size -= guest_mmu_size;
		cvmvmconfig &= ~CVMVMCONF_RMMUSIZEM1;
		cvmvmconfig |= mmu_size - 1;
		write_c0_cvmvmconfig(cvmvmconfig);

		/* Update our records */
		current_cpu_data.tlbsize = mmu_size;
		current_cpu_data.tlbsizevtlb = mmu_size;
		current_cpu_data.guest.tlbsize = guest_mmu_size;

		/* Flush moved entries in new (guest) context */
		kvm_vz_local_flush_guesttlb_all();
		break;
	default:
		/*
		 * ImgTec cores tend to use a shared root/guest TLB. To avoid
		 * overlap of root wired and guest entries, the guest TLB may
		 * need resizing.
		 */
		mmu_size = current_cpu_data.tlbsizevtlb;
		ftlb_size = current_cpu_data.tlbsize - mmu_size;

		/* Try switching to maximum guest VTLB size for flush */
		guest_mmu_size = kvm_vz_resize_guest_vtlb(mmu_size);
		current_cpu_data.guest.tlbsize = guest_mmu_size + ftlb_size;
		kvm_vz_local_flush_guesttlb_all();

		/*
		 * Reduce to make space for root wired entries and at least 2
		 * root non-wired entries. This does assume that long-term wired
		 * entries won't be added later.
		 */
		guest_mmu_size = mmu_size - num_wired_entries() - 2;
		guest_mmu_size = kvm_vz_resize_guest_vtlb(guest_mmu_size);
		current_cpu_data.guest.tlbsize = guest_mmu_size + ftlb_size;

		/*
		 * Write the VTLB size, but if another CPU has already written,
		 * check it matches or we won't provide a consistent view to the
		 * guest. If this ever happens it suggests an asymmetric number
		 * of wired entries.
		 */
		if (cmpxchg(&kvm_vz_guest_vtlb_size, 0, guest_mmu_size) &&
		    WARN(guest_mmu_size != kvm_vz_guest_vtlb_size,
			 "Available guest VTLB size mismatch"))
			return -EINVAL;
		break;
	}

	/*
	 * Enable virtualization features granting guest direct control of
	 * certain features:
	 * CP0=1:	Guest coprocessor 0 context.
	 * AT=Guest:	Guest MMU.
	 * CG=1:	Hit (virtual address) CACHE operations (optional).
	 * CF=1:	Guest Config registers.
	 * CGI=1:	Indexed flush CACHE operations (optional).
	 */
	write_c0_guestctl0(MIPS_GCTL0_CP0 |
			   (MIPS_GCTL0_AT_GUEST << MIPS_GCTL0_AT_SHIFT) |
			   MIPS_GCTL0_CG | MIPS_GCTL0_CF);
	if (cpu_has_guestctl0ext) {
		if (current_cpu_type() != CPU_LOONGSON64)
			set_c0_guestctl0ext(MIPS_GCTL0EXT_CGI);
		else
			clear_c0_guestctl0ext(MIPS_GCTL0EXT_CGI);
	}

	if (cpu_has_guestid) {
		write_c0_guestctl1(0);
		kvm_vz_local_flush_roottlb_all_guests();

		GUESTID_MASK = current_cpu_data.guestid_mask;
		GUESTID_FIRST_VERSION = GUESTID_MASK + 1;
		GUESTID_VERSION_MASK = ~GUESTID_MASK;

		current_cpu_data.guestid_cache = GUESTID_FIRST_VERSION;
	}

	/* clear any pending injected virtual guest interrupts */
	if (cpu_has_guestctl2)
		clear_c0_guestctl2(0x3f << 10);

#ifdef CONFIG_CPU_LOONGSON64
	/* Control guest CCA attribute */
	if (cpu_has_csr())
		csr_writel(csr_readl(0xffffffec) | 0x1, 0xffffffec);
#endif

	return 0;
}

static void kvm_vz_hardware_disable(void)
{
	u64 cvmvmconfig;
	unsigned int mmu_size;

	/* Flush any remaining guest TLB entries */
	kvm_vz_local_flush_guesttlb_all();

	switch (current_cpu_type()) {
	case CPU_CAVIUM_OCTEON3:
		/*
		 * Allocate whole TLB for root. Existing guest TLB entries will