diff --git a/arch/x64/loader.ld b/arch/x64/loader.ld
index 8980f6ab02a63be9c9b968e2e8c1be6d9618cdbf..1b528534b94d9eaa8feea97269cd784974905356 100644
--- a/arch/x64/loader.ld
+++ b/arch/x64/loader.ld
@@ -25,6 +25,11 @@ SECTIONS
     _init_array_start = .;
     .init_array : { *(SORT_BY_INIT_PRIORITY(.init_array.*)) } :text
     _init_array_end = .;
+    .percpu : {
+        _percpu_start = .;
+        *(.percpu)
+        _percpu_end = .;
+    }
     .tls_template_start = .;
     .tdata : { *(.tdata .tdata.* .gnu.linkonce.td.*) } :tls :text
     .tbss : { *(.tbss .tbss.* .gnu.linkonce.tb.*) } :tls :text
diff --git a/arch/x64/smp.cc b/arch/x64/smp.cc
index 8d3484107d45774a46b651cf8c3e6200de476160..bdd79f58647ec84670b45ba8be4f5231048fd2ad 100644
--- a/arch/x64/smp.cc
+++ b/arch/x64/smp.cc
@@ -46,8 +46,7 @@ void parse_madt()
             if (!(lapic->LapicFlags & ACPI_MADT_ENABLED)) {
                 break;
             }
-            auto c = new sched::cpu;
-            c->id = idgen++;
+            auto c = new sched::cpu(idgen++);
             c->arch.apic_id = lapic->Id;
             c->arch.acpi_id = lapic->ProcessorId;
             c->arch.initstack.next = smp_stack_free;
diff --git a/bootfs.manifest b/bootfs.manifest
index cf23fd92bf4c407b28286b866e3329beb195cb89..69c7f9728c38e7f7d8ad1e981a290da9f035470f 100644
--- a/bootfs.manifest
+++ b/bootfs.manifest
@@ -101,6 +101,7 @@
 /usr/lib/&/jni/elf-loader.so: java/&
 /usr/lib/&/jni/networking.so: java/&
 /usr/lib/&/jni/stty.so: java/&
+/usr/lib/&/jni/tracepoint.so: java/&
 /tools/ifconfig.so: ./tools/ifconfig/ifconfig.so
 /tools/lsroute.so: ./tools/route/lsroute.so
 /console/util.js: ../../console/util.js
@@ -120,4 +121,5 @@
 /console/cli.js: ../../console/cli.js
 /console/init.js: ../../console/init.js
 /console/md5sum.js: ../../console/md5sum.js
+/&/console/perf.js: ../../&
 /&/etc/hosts: ../../static/&
diff --git a/build.mak b/build.mak
index 03865bc1df84c19efd453f5eec2cf6c2d93757f7..77d6d6de7a09bd1cf654a42305233ee15ad521fd 100644
--- a/build.mak
+++ b/build.mak
@@ -437,6 +437,8 @@ objects += core/trace.o
 objects += core/poll.o
 objects += core/select.o
 objects += core/power.o
+objects += core/percpu.o
+objects += core/per-cpu-counter.o
 
 include $(src)/fs/build.mak
 include $(src)/libc/build.mak
@@ -490,7 +492,7 @@ usr.img: usr.manifest
 		sh $(src)/scripts/mkromfs.sh, MKROMFS $@)
 
 jni = java/jni/balloon.so java/jni/elf-loader.so java/jni/networking.so \
-	java/jni/stty.so
+	java/jni/stty.so java/jni/tracepoint.so
 $(jni): INCLUDES += -I /usr/lib/jvm/java/include -I /usr/lib/jvm/java/include/linux/
 
 bootfs.bin: scripts/mkbootfs.py bootfs.manifest $(tests) $(tools) $(jni) \
diff --git a/console/cli.js b/console/cli.js
index 019eef2f793f87deecdf258419b7c355d4fd11a8..850ea7d94f60a4925b064c4893aa3c5d23f51f10 100644
--- a/console/cli.js
+++ b/console/cli.js
@@ -1,5 +1,4 @@
 importPackage(java.io);
-importPackage(java.lang);
 importPackage(com.cloudius.util);
 importPackage(com.cloudius.cli.util);
 
@@ -18,6 +17,7 @@ load("/console/arp.js");
 load("/console/md5sum.js");
 load("/console/route.js");
 load("/console/java.js");
+load("/console/perf.js");
 
 // Commands
 var _commands = new Array();
@@ -33,10 +33,13 @@ _commands["arp"] = arp_cmd;
 _commands["route"] = route_cmd;
 _commands["md5sum"] = md5sum_cmd;
 _commands["java"] = java_cmd;
+_commands["perf"] = perf_cmd;
 
 // Create interface to networking functions
 var networking_interface = new Networking();
 
+var System = java.lang.System
+
 // I/O
 var _reader = new BufferedReader( new InputStreamReader(System['in']) );
 var _writer = new BufferedWriter( new OutputStreamWriter(System['out']));
diff --git a/console/perf.js b/console/perf.js
new file mode 100644
index 0000000000000000000000000000000000000000..3f04addd07686a1726a9cad6a92c115f8f7cb7d8
--- /dev/null
+++ b/console/perf.js
@@ -0,0 +1,104 @@
+
+
+var perf_cmd = {
+    invoke: function(args) {
+        args.shift()
+        var cmd = args.shift()
+        if (cmd in this.subcommands) {
+            this.subcommands[cmd].invoke(args)
+        } else {
+            this.help([])
+        }
+    },
+    help: function(args) {
+        write_string('usage:\n\n')
+        for (var k in this.subcommands) {
+            write_string('  perf ' + this.subcommands[k].usage + '\n')
+        }
+    },
+    list: function(args) {
+        write_string('available tracpoints:\n\n')
+        trace = Packages.com.cloudius.trace
+        all = trace.Tracepoint.list()
+        for (var i = 0; i < all.size(); ++i) {
+            var tp = all.get(i)
+            write_string('    ' + String(tp.getName()))
+            write_char('\n')
+        }
+    },
+    stat: function(args) {
+        var pkg = Packages.com.cloudius.trace
+        var counters = []
+        for (var i in args) {
+            var x = args[i]
+            var m = /^(([^=]+)=)?(.+)$/.exec(x)
+            var tag = m[2]
+            var name = m[3]
+            if (!tag) {
+                tag = name
+            }
+            try {
+                var tp = new pkg.Tracepoint(name)
+                var counter = new pkg.Counter(tp)
+                counters.push({
+                    tag: tag,
+                    counter: counter,
+                    width: Math.max(8, tag.length + 2),
+                    last: 0,
+                })
+            } catch (err) {
+                write_string('bad tracepoint "' + name + '"\n')
+                return
+            }
+        }
+        var titles = function() {
+            for (var i in counters) {
+                var c = counters[i]
+                for (var j = c.tag.length; j < c.width; ++j) {
+                    write_string(' ')
+                }
+                write_string(c.tag)
+            }
+            write_string('\n')
+        }
+        var show = function() {
+            for (var i in counters) {
+                var ctr = counters[i]
+                var last = ctr.last
+                ctr.last = ctr.counter.read()
+                var delta = ctr.last - last
+                delta = delta.toString()
+                for (var j = delta.length; j < ctr.width; ++j) {
+                    write_string(' ')
+                }
+                write_string(delta)
+            }
+            write_string('\n')
+        }
+        var line = 0
+        while (true) {
+            if (line++ % 25 == 0) {
+                titles()
+            }
+            show()
+            flush()
+            java.lang.Thread.sleep(1000)
+        }
+    },
+    subcommands: {
+        list: {
+            invoke: function(args) { this.parent.list(args) },
+            usage: 'list',
+        },
+        stat: {
+            invoke: function(args) { this.parent.stat(args) },
+            usage: 'stat [[tag=]tracepoint]...',
+        },
+    },
+    init: function() {
+        for (var k in this.subcommands) {
+            this.subcommands[k].parent = this
+        }
+    },
+}
+
diff --git a/core/per-cpu-counter.cc b/core/per-cpu-counter.cc
new file mode 100644
index 0000000000000000000000000000000000000000..c63080e1dbb61d8048145015794d410b09ae65a6
--- /dev/null
+++ b/core/per-cpu-counter.cc
@@ -0,0 +1,68 @@
+#include <osv/per-cpu-counter.hh>
+#include <osv/mutex.h>
+#include <debug.hh>
+
+PERCPU(ulong*, per_cpu_counter::_counters);
+
+namespace {
+
+const size_t max_counters = 1000;   // FIXME: allow auto-expand later
+
+static std::vector<bool> used_indices(max_counters);
+mutex mtx;
+
+unsigned allocate_index()
+{
+    std::lock_guard<mutex> guard{mtx};
+    auto i = std::find(used_indices.begin(), used_indices.end(), false);
+    if (i == used_indices.end()) {
+        abort("out of per-cpu counters");
+    }
+    *i = true;
+    return i - used_indices.begin();
+}
+
+void free_index(unsigned index)
+{
+    std::lock_guard<mutex> guard{mtx};
+    assert(used_indices[index]);
+    used_indices[index] = false;
+}
+
+}
+
+per_cpu_counter::per_cpu_counter()
+    : _index(allocate_index())
+{
+    for (auto cpu : sched::cpus) {
+        (*_counters.for_cpu(cpu))[_index] = 0;
+    }
+}
+
+per_cpu_counter::~per_cpu_counter()
+{
+    free_index(_index);
+}
+
+void per_cpu_counter::increment()
+{
+    sched::preempt_disable();
+    ++(*_counters)[_index];
+    sched::preempt_enable();
+}
+
+ulong per_cpu_counter::read()
+{
+    ulong sum = 0;
+    for (auto cpu : sched::cpus) {
+        sum += (*_counters.for_cpu(cpu))[_index];
+    }
+    return sum;
+}
+
+void per_cpu_counter::init_on_cpu()
+{
+    *_counters = new ulong[max_counters];
+}
+
+sched::cpu::notifier per_cpu_counter::_cpu_notifier(per_cpu_counter::init_on_cpu);
diff --git a/core/percpu.cc b/core/percpu.cc
new file mode 100644
index 0000000000000000000000000000000000000000..29e93684e549432a4a4c79c6cac0e6c246e393b6
--- /dev/null
+++ b/core/percpu.cc
@@ -0,0 +1,14 @@
+#include <osv/percpu.hh>
+#include <string.h>
+
+std::vector<void*> percpu_base{64};  // FIXME: move to sched::cpu
+
+extern char _percpu_start[], _percpu_end[];
+
+void percpu_init(unsigned cpu)
+{
+    assert(!percpu_base[cpu]);
+    auto size = _percpu_end - _percpu_start;
+    percpu_base[cpu] = malloc(size);
+    memcpy(percpu_base[cpu], _percpu_start, size);
+}
diff --git a/core/sched.cc b/core/sched.cc
index ddd37947f282f2e9a960b630a0677d996fc6fd57..04b5a793f1f64b212ced39f032615e9a4e79a8da 100644
--- a/core/sched.cc
+++ b/core/sched.cc
@@ -10,6 +10,7 @@
 #include "interrupt.hh"
 #include "smp.hh"
 #include "osv/trace.hh"
+#include <osv/percpu.hh>
 
 namespace sched {
 
@@ -34,6 +35,9 @@ inter_processor_interrupt wakeup_ipi{[] {}};
 constexpr u64 vruntime_bias = 4_ms;
 constexpr u64 max_slice = 10_ms;
 
+mutex cpu::notifier::_mtx;
+std::list<cpu::notifier*> cpu::notifier::_notifiers __attribute__((init_priority(300)));
+
 }
 
 #include "arch-switch.hh"
@@ -51,9 +55,11 @@ private:
     thread _thread;
 };
 
-cpu::cpu()
-    : idle_thread([this] { idle(); }, thread::attr(this))
+cpu::cpu(unsigned _id)
+    : id(_id)
+    , idle_thread([this] { idle(); }, thread::attr(this))
 {
+    percpu_init(id);
 }
 
 void cpu::schedule(bool yield)
@@ -190,6 +196,7 @@ unsigned cpu::load()
 
 void cpu::load_balance()
 {
+    notifier::fire();
     timer tmr(*thread::current());
     while (true) {
         tmr.set(clock::get()->time() + 100_ms);
@@ -225,6 +232,26 @@ void cpu::load_balance()
     }
 }
 
+cpu::notifier::notifier(std::function<void ()> cpu_up)
+    : _cpu_up(cpu_up)
+{
+    with_lock(_mtx, [this] { _notifiers.push_back(this); });
+}
+
+cpu::notifier::~notifier()
+{
+    with_lock(_mtx, [this] { _notifiers.remove(this); });
+}
+
+void cpu::notifier::fire()
+{
+    with_lock(_mtx, [] {
+        for (auto n : _notifiers) {
+            n->_cpu_up();
+        }
+    });
+}
+
 void schedule(bool yield)
 {
     cpu::current()->schedule(yield);
diff --git a/core/trace.cc b/core/trace.cc
index 853cb00eba25b2c1b765ce872db4dadd5af796e1..dd66fa4b69ef001afe265b674d86f22edf51f87a 100644
--- a/core/trace.cc
+++ b/core/trace.cc
@@ -4,6 +4,7 @@
 #include <atomic>
 #include <regex>
 #include <boost/algorithm/string/replace.hpp>
+#include <boost/range/algorithm/remove.hpp>
 #include <debug.hh>
 
 tracepoint<1, void*, void*> trace_function_entry("function entry", "fn %p caller %p");
@@ -95,10 +96,23 @@ tracepoint_base::~tracepoint_base()
 
 void tracepoint_base::enable()
 {
-    if (enabled) {
-        return;
+    logging = true;
+    update();
+}
+
+void tracepoint_base::update()
+{
+    bool new_active = logging || !probes.empty();
+    if (new_active && !active) {
+        activate();
+    } else if (!new_active && active) {
+        deactivate();
     }
-    enabled = true;
+}
+
+void tracepoint_base::activate()
+{
+    active = true;
     for (auto& tps : tracepoint_patch_sites) {
         if (id == tps.id) {
             auto dst = static_cast<char*>(tps.slow_path);
@@ -110,6 +124,28 @@ void tracepoint_base::enable()
     }
 }
 
+void tracepoint_base::deactivate()
+{
+    active = false;
+    for (auto& tps : tracepoint_patch_sites) {
+        if (id == tps.id) {
+            auto p = static_cast<u8*>(tps.patch_site);
+            // FIXME: can fail on smp.
+            p[0] = 0x0f;
+            p[1] = 0x1f;
+            p[2] = 0x44;
+            p[3] = 0x00;
+            p[4] = 0x00;
+        }
+    }
+}
+
+void tracepoint_base::run_probes() {
+    for (auto probe : probes) {
+        probe->hit();
+    }
+}
+
 void tracepoint_base::try_enable()
 {
     for (auto& re : enabled_tracepoint_regexs) {
@@ -119,6 +155,21 @@ void tracepoint_base::try_enable()
     }
 }
 
+void tracepoint_base::add_probe(probe* p)
+{
+    // FIXME: locking
+    probes.push_back(p);
+    update();
+}
+
+void tracepoint_base::del_probe(probe* p)
+{
+    // FIXME: locking
+    auto i = boost::remove(probes, p);
+    probes.erase(i, probes.end());
+    update();
+}
+
 std::unordered_set<tracepoint_id>& tracepoint_base::known_ids()
 {
     // since tracepoints are constructed in global scope, use
diff --git a/drivers/kvmclock.cc b/drivers/kvmclock.cc
index 54256183bc0e727905ad9a04c9bb6ed527e19936..2fd37423c666c2978163300fb22d09b2de6fb39a 100644
--- a/drivers/kvmclock.cc
+++ b/drivers/kvmclock.cc
@@ -5,6 +5,7 @@
 #include "string.h"
 #include "cpuid.hh"
 #include "barrier.hh"
+#include <osv/percpu.hh>
 
 class kvmclock : public clock {
 private:
@@ -29,26 +30,45 @@ public:
 private:
     u64 wall_clock_boot();
     u64 system_time();
+    static void setup_cpu();
 private:
+    static bool _smp_init;
     pvclock_wall_clock* _wall;
-    pvclock_vcpu_time_info* _sys;  // FIXME: make percpu
+    static PERCPU(pvclock_vcpu_time_info, _sys);
+    sched::cpu::notifier cpu_notifier;
 };
 
+bool kvmclock::_smp_init = false;
+PERCPU(kvmclock::pvclock_vcpu_time_info, kvmclock::_sys);
+
 kvmclock::kvmclock()
+    : cpu_notifier(&kvmclock::setup_cpu)
 {
     _wall = new kvmclock::pvclock_wall_clock;
-    _sys = new kvmclock::pvclock_vcpu_time_info;
     memset(_wall, 0, sizeof(*_wall));
-    memset(_sys, 0, sizeof(*_sys));
     processor::wrmsr(msr::KVM_WALL_CLOCK_NEW, mmu::virt_to_phys(_wall));
-    // FIXME: on each cpu
-    processor::wrmsr(msr::KVM_SYSTEM_TIME_NEW, mmu::virt_to_phys(_sys) | 1);
+}
+
+void kvmclock::setup_cpu()
+{
+    memset(&*_sys, 0, sizeof(*_sys));
+    processor::wrmsr(msr::KVM_SYSTEM_TIME_NEW, mmu::virt_to_phys(&*_sys) | 1);
+    _smp_init = true;
 }
 
 u64 kvmclock::time()
 {
-    // FIXME: disable interrupts
-    return wall_clock_boot() + system_time();
+    sched::preempt_disable();
+    auto r = wall_clock_boot();
+    // Due to problems in init order dependencies (the clock depends
+    // on the scheduler, for percpu initialization, and vice-versa, for
+    // idle thread initialization, don't loop up system time until at least
+    // one cpu is initialized.
+    if (_smp_init) {
+        r += system_time();
+    }
+    sched::preempt_enable();
+    return r;
 }
 
 u64 kvmclock::wall_clock_boot()
@@ -69,22 +89,23 @@ u64 kvmclock::system_time()
 {
     u32 v1, v2;
     u64 time;
+    auto sys = &*_sys;  // avoid recaclulating address each access
     do {
-        v1 = _sys->version;
+        v1 = sys->version;
         barrier();
-        time = processor::rdtsc() - _sys->tsc_timestamp;
-        if (_sys->tsc_shift >= 0) {
-            time <<= _sys->tsc_shift;
+        time = processor::rdtsc() - sys->tsc_timestamp;
+        if (sys->tsc_shift >= 0) {
+            time <<= sys->tsc_shift;
         } else {
-            time >>= -_sys->tsc_shift;
+            time >>= -sys->tsc_shift;
         }
         asm("mul %1; shrd $32, %%rdx, %0"
                 : "+a"(time)
-                : "rm"(u64(_sys->tsc_to_system_mul))
+                : "rm"(u64(sys->tsc_to_system_mul))
                 : "rdx");
-        time += _sys->system_time;
+        time += sys->system_time;
         barrier();
-        v2 = _sys->version;
+        v2 = sys->version;
     } while (v1 != v2);
     return time;
 }
diff --git a/include/osv/per-cpu-counter.hh b/include/osv/per-cpu-counter.hh
new file mode 100644
index 0000000000000000000000000000000000000000..4f7b81ce35563fa2fb1d855924b085841fbde4cf
--- /dev/null
+++ b/include/osv/per-cpu-counter.hh
@@ -0,0 +1,25 @@
+#ifndef PER_CPU_COUNTER_HH_
+#define PER_CPU_COUNTER_HH_
+
+#include <osv/types.h>
+#include <sched.hh>
+#include <osv/percpu.hh>
+#include <sched.hh>
+#include <vector>
+#include <memory>
+
+class per_cpu_counter {
+public:
+    explicit per_cpu_counter();
+    ~per_cpu_counter();
+    void increment();
+    ulong read();
+private:
+    unsigned _index;
+private:
+    static percpu<ulong*> _counters;
+    static sched::cpu::notifier _cpu_notifier;
+    static void init_on_cpu();
+};
+
+#endif /* PER_CPU_COUNTER_HH_ */
diff --git a/include/osv/percpu.hh b/include/osv/percpu.hh
new file mode 100644
index 0000000000000000000000000000000000000000..1df6a1ef0b4df562c57b8d0bf4453350c943b6e0
--- /dev/null
+++ b/include/osv/percpu.hh
@@ -0,0 +1,37 @@
+#ifndef PERCPU_HH_
+#define PERCPU_HH_
+
+#include <sched.hh>
+
+extern char _percpu_start[];
+
+extern std::vector<void*> percpu_base;  // FIXME: move to sched::cpu
+
+template <typename T>
+class percpu {
+public:
+    constexpr percpu() {}
+    T* operator->() {
+        return for_cpu(sched::cpu::current());
+    }
+    T& operator*() {
+        return *addr(sched::cpu::current());
+    }
+    T* for_cpu(sched::cpu* cpu) {
+        return addr(cpu);
+    }
+private:
+    T *addr(sched::cpu* cpu) {
+        void* base = percpu_base[cpu->id];
+        size_t offset = reinterpret_cast<char*>(&_var) - _percpu_start;
+        return reinterpret_cast<T*>(base + offset);
+    }
+private:
+    T _var;
+};
+
+#define PERCPU(type, var) percpu<type> var __attribute__((section(".percpu")))
+
+void percpu_init(unsigned cpu);
+
+#endif /* PERCPU_HH_ */
diff --git a/include/osv/trace.hh b/include/osv/trace.hh
index db12ba06ea673e1e20d24e0f3b1e77d3e98de0d9..26cf57b599b5f81bd5858b70f4802574237c922e 100644
--- a/include/osv/trace.hh
+++ b/include/osv/trace.hh
@@ -210,16 +210,22 @@ struct serializer<N, N, args...> {
 typedef std::tuple<const std::type_info*, unsigned long> tracepoint_id;
 
 class tracepoint_base {
+public:
+    struct probe {
+        virtual ~probe() {}
+        virtual void hit() = 0;
+    };
 public:
     explicit tracepoint_base(unsigned _id, const std::type_info& _tp_type,
                              const char* _name, const char* _format);
     ~tracepoint_base();
     void enable();
+    void add_probe(probe* p);
+    void del_probe(probe* p);
     tracepoint_id id;
     const char* name;
     const char* format;
     u64 sig;
-    bool enabled = false;
     typedef boost::intrusive::list_member_hook<> tp_list_link_type;
     tp_list_link_type tp_list_link;
     static boost::intrusive::list<
@@ -229,8 +235,16 @@ public:
                                       &tracepoint_base::tp_list_link>,
         boost::intrusive::constant_time_size<false>
         > tp_list;
+protected:
+    bool active = false; // logging || !probes.empty()
+    bool logging = false;
+    std::vector<probe*> probes;
+    void run_probes();
 private:
     void try_enable();
+    void activate();
+    void deactivate();
+    void update();
     static std::unordered_set<tracepoint_id>& known_ids();
 };
 
@@ -267,22 +281,30 @@ public:
         trace_slow_path(assign(as...));
     }
     void trace_slow_path(std::tuple<s_args...> as) __attribute__((cold)) {
-        if (enabled) {
+        if (active) {
             arch::irq_flag_notrace irq;
             irq.save();
             arch::irq_disable_notrace();
-            auto tr = allocate_trace_record(size());
-            tr->tp = this;
-            tr->thread = sched::thread::current();
-            tr->time = clock::get()->time();
-            tr->cpu = -1;
-            if (tr->thread) {
-                tr->cpu = tr->thread->tcpu()->id;
-            }
-            serialize(tr->buffer, as);
+#undef log
+            log(as);
+            run_probes();
             irq.restore();
         }
     }
+    void log(const std::tuple<s_args...>& as) {
+        if (!logging) {
+            return;
+        }
+        auto tr = allocate_trace_record(size());
+        tr->tp = this;
+        tr->thread = sched::thread::current();
+        tr->time = clock::get()->time();
+        tr->cpu = -1;
+        if (tr->thread) {
+            tr->cpu = tr->thread->tcpu()->id;
+        }
+        serialize(tr->buffer, as);
+    }
     void serialize(void* buffer, std::tuple<s_args...> as) {
         return serializer<0, sizeof...(s_args), s_args...>::write(buffer, 0, as);
     }
diff --git a/include/sched.hh b/include/sched.hh
index 76b0be6bb5a22641c697814cd4635a6842a149db..3e8474465dc9365507f3fdc408e27fb98bdd6d74 100644
--- a/include/sched.hh
+++ b/include/sched.hh
@@ -12,6 +12,7 @@
 #include <osv/mutex.h>
 #include <atomic>
 #include "osv/lockless-queue.hh"
+#include <list>
 
 extern "C" {
 void smp_main();
@@ -273,7 +274,7 @@ typedef bi::rbtree<thread,
                   > runqueue_type;
 
 struct cpu {
-    explicit cpu();
+    explicit cpu(unsigned id);
     unsigned id;
     struct arch_cpu arch;
     thread* bringup_thread;
@@ -294,6 +295,20 @@ struct cpu {
     unsigned load();
     void reschedule_from_interrupt(bool preempt = false);
     void enqueue(thread& t, u64 now);
+    class notifier;
+};
+
+class cpu::notifier {
+public:
+    explicit notifier(std::function<void ()> cpu_up);
+    ~notifier();
+private:
+    static void fire();
+private:
+    std::function<void ()> _cpu_up;
+    static mutex _mtx;
+    static std::list<notifier*> _notifiers;
+    friend class cpu;
 };
 
 void preempt();
diff --git a/java/cloudius/com/cloudius/trace/Counter.java b/java/cloudius/com/cloudius/trace/Counter.java
new file mode 100644
index 0000000000000000000000000000000000000000..6ad6970a039baefb22949907a962c268496a2999
--- /dev/null
+++ b/java/cloudius/com/cloudius/trace/Counter.java
@@ -0,0 +1,34 @@
+package com.cloudius.trace;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+public class Counter implements Closeable {
+
+	private long handle;
+	
+	public Counter(Tracepoint tp) {
+		handle = tp.createCounter();
+	}
+
+	public long read() {
+		return Tracepoint.readCounter(handle);
+	}
+	
+	@Override
+	public void finalize() {
+		try {
+			close();
+		} catch (IOException e) {
+			// nothing we can do.
+		}
+	}
+
+	@Override
+	public void close() throws IOException {
+		if (handle != 0) {
+			Tracepoint.destroyCounter(handle);
+			handle = 0;
+		}
+	}
+}
diff --git a/java/cloudius/com/cloudius/trace/Tracepoint.java b/java/cloudius/com/cloudius/trace/Tracepoint.java
new file mode 100644
index 0000000000000000000000000000000000000000..2fbefcfbec3d61df498b8d996b5dbf31e150b9e3
--- /dev/null
+++ b/java/cloudius/com/cloudius/trace/Tracepoint.java
@@ -0,0 +1,59 @@
+package com.cloudius.trace;
+
+import com.cloudius.*;
+import java.util.*;
+
+public class Tracepoint {
+
+	static {
+		Config.loadJNI("tracepoint.so");
+	}
+	
+	private String name;
+	private long handle;
+	
+	Tracepoint(long handle) {
+		this.handle = handle;
+	}
+	
+	public Tracepoint(String name) {
+		this.handle = findByName(name);
+	}
+	
+	public String getName() {
+		return doGetName(handle);
+	}
+	
+	public void enable() {
+		doEnable(handle);
+	}
+
+	public static List<Tracepoint> list() {
+		long[] handles = doList();
+		System.out.flush();
+		List<Tracepoint> ret = new ArrayList<Tracepoint>(handles.length);
+		for (long handle : handles) {
+			ret.add(new Tracepoint(handle));
+		}
+		return ret;
+	}
+		
+	long createCounter() {
+		return doCreateCounter(handle);
+	}
+	
+	native static long[] doList();
+	
+	native static long findByName(String name);
+	
+	native static void doEnable(long handle);
+	
+	native static String doGetName(long handle);
+	
+	native static long doCreateCounter(long handle);
+	
+	native static void destroyCounter(long handle); 
+	
+	native static long readCounter(long handle); 
+	
+}
diff --git a/java/jni/tracepoint.cc b/java/jni/tracepoint.cc
new file mode 100644
index 0000000000000000000000000000000000000000..5820cdec6d42a91ea08461562da3db8564d60488
--- /dev/null
+++ b/java/jni/tracepoint.cc
@@ -0,0 +1,89 @@
+#include "tracepoint.hh"
+#include <osv/trace.hh>
+#include <osv/per-cpu-counter.hh>
+#include <debug.hh>
+
+static std::string get_string(JNIEnv* jni, jstring s)
+{
+    auto p = jni->GetStringUTFChars(s, nullptr);
+    std::string ret(p);
+    jni->ReleaseStringUTFChars(s, p);
+    return ret;
+}
+
+JNIEXPORT jlongArray JNICALL Java_com_cloudius_trace_Tracepoint_doList
+  (JNIEnv *jni, jclass klass)
+{
+    auto nr = tracepoint_base::tp_list.size();
+    auto a = jni->NewLongArray(nr);
+    size_t idx = 0;
+    for (auto& tp : tracepoint_base::tp_list) {
+        jlong handle = jlong(reinterpret_cast<uintptr_t>(&tp));
+        jni->SetLongArrayRegion(a, idx++, 1, &handle);
+    };
+    return a;
+}
+
+JNIEXPORT jlong JNICALL Java_com_cloudius_trace_Tracepoint_findByName
+  (JNIEnv *jni, jclass klass, jstring name)
+{
+    auto n = get_string(jni, name);
+    for (auto& tp : tracepoint_base::tp_list) {
+        if (n == tp.name) {
+            return reinterpret_cast<uintptr_t>(&tp);
+        }
+    }
+    auto re = jni->FindClass("java/lang/RuntimeException");
+    jni->ThrowNew(re, "Cannot find tracepoint");
+    return 0;
+}
+
+JNIEXPORT void JNICALL Java_com_cloudius_trace_Tracepoint_doEnable
+  (JNIEnv *jni, jclass klass, jlong handle)
+{
+    auto tp = reinterpret_cast<tracepoint_base*>(handle);
+    tp->enable();
+}
+
+JNIEXPORT jstring JNICALL Java_com_cloudius_trace_Tracepoint_doGetName
+  (JNIEnv *jni, jclass klass, jlong handle)
+{
+    auto tp = reinterpret_cast<tracepoint_base*>(handle);
+    return jni->NewStringUTF(tp->name);
+}
+
+class tracepoint_counter : public tracepoint_base::probe {
+public:
+    explicit tracepoint_counter(tracepoint_base& tp) : _tp(tp) {
+        _tp.add_probe(this);
+    }
+    virtual ~tracepoint_counter() { _tp.del_probe(this); }
+    virtual void hit() { _counter.increment(); }
+    ulong read() { return _counter.read(); }
+private:
+    tracepoint_base& _tp;
+    per_cpu_counter _counter;
+};
+
+JNIEXPORT jlong JNICALL Java_com_cloudius_trace_Tracepoint_doCreateCounter
+  (JNIEnv *jni, jclass klass, jlong handle)
+{
+    auto tp = reinterpret_cast<tracepoint_base*>(handle);
+    auto c = new tracepoint_counter(*tp);
+    return reinterpret_cast<jlong>(c);
+}
+
+JNIEXPORT void JNICALL Java_com_cloudius_trace_Tracepoint_destroyCounter
+  (JNIEnv *jni, jclass klazz, jlong handle)
+{
+    auto c = reinterpret_cast<tracepoint_counter*>(handle);
+    delete c;
+}
+
+JNIEXPORT jlong JNICALL Java_com_cloudius_trace_Tracepoint_readCounter
+  (JNIEnv *jni, jclass klass, jlong handle)
+{
+    auto c = reinterpret_cast<tracepoint_counter*>(handle);
+    return c->read();
+}
+
diff --git a/java/jni/tracepoint.hh b/java/jni/tracepoint.hh
new file mode 100644
index 0000000000000000000000000000000000000000..e375bfc7cd013a04e4c28732c1c18c747a6d184b
--- /dev/null
+++ b/java/jni/tracepoint.hh
@@ -0,0 +1,69 @@
+/* DO NOT EDIT THIS FILE - it is machine generated */
+#include <jni.h>
+/* Header for class com_cloudius_trace_Tracepoint */
+
+#ifndef _Included_com_cloudius_trace_Tracepoint
+#define _Included_com_cloudius_trace_Tracepoint
+#ifdef __cplusplus
+extern "C" {
+#endif
+/*
+ * Class:     com_cloudius_trace_Tracepoint
+ * Method:    doList
+ * Signature: ()[J
+ */
+JNIEXPORT jlongArray JNICALL Java_com_cloudius_trace_Tracepoint_doList
+  (JNIEnv *, jclass);
+
+/*
+ * Class:     com_cloudius_trace_Tracepoint
+ * Method:    findByName
+ * Signature: (Ljava/lang/String;)J
+ */
+JNIEXPORT jlong JNICALL Java_com_cloudius_trace_Tracepoint_findByName
+  (JNIEnv *, jclass, jstring);
+
+/*
+ * Class:     com_cloudius_trace_Tracepoint
+ * Method:    doEnable
+ * Signature: (J)V
+ */
+JNIEXPORT void JNICALL Java_com_cloudius_trace_Tracepoint_doEnable
+  (JNIEnv *, jclass, jlong);
+
+/*
+ * Class:     com_cloudius_trace_Tracepoint
+ * Method:    doGetName
+ * Signature: (J)Ljava/lang/String;
+ */
+JNIEXPORT jstring JNICALL Java_com_cloudius_trace_Tracepoint_doGetName
+  (JNIEnv *, jclass, jlong);
+
+/*
+ * Class:     com_cloudius_trace_Tracepoint
+ * Method:    doCreateCounter
+ * Signature: (J)J
+ */
+JNIEXPORT jlong JNICALL Java_com_cloudius_trace_Tracepoint_doCreateCounter
+  (JNIEnv *, jclass, jlong);
+
+/*
+ * Class:     com_cloudius_trace_Tracepoint
+ * Method:    destroyCounter
+ * Signature: (J)V
+ */
+JNIEXPORT void JNICALL Java_com_cloudius_trace_Tracepoint_destroyCounter
+  (JNIEnv *, jclass, jlong);
+
+/*
+ * Class:     com_cloudius_trace_Tracepoint
+ * Method:    readCounter
+ * Signature: (J)J
+ */
+JNIEXPORT jlong JNICALL Java_com_cloudius_trace_Tracepoint_readCounter
+  (JNIEnv *, jclass, jlong);
+
+#ifdef __cplusplus
+}
+#endif
+#endif