VDSO constants

h2o/boot/Cargo.toml

@@ -12,7 +12,7 @@ bitop_ex = {path = "../libs/bitop_ex"}
 minfo = {path = "../libs/minfo"}
 paging = {path = "../libs/paging"}
 # External crates
-goblin = {version = "0.4", default-features = false, features = ["elf32", "elf64", "endian_fd"]}
+goblin = {version = "0.5", default-features = false, features = ["elf32", "elf64", "endian_fd"]}
 log = "0.4"
 raw-cpuid = "9.0"
 static_assertions = "1.1"

h2o/kernel/Cargo.toml

@@ -35,7 +35,7 @@ crossbeam-queue = {version = "0.3", default-features = false, features = ["alloc
 crossbeam-utils = {version = "0.8", default-features = false}
 cty = "0.2"
 derive_builder = {version = "0.10", default-features = false}
-goblin = {version = "0.4", default-features = false, features = ["elf32", "elf64", "endian_fd"]}
+goblin = {version = "0.5", default-features = false, features = ["elf32", "elf64", "endian_fd"]}
 log = "0.4"
 modular-bitfield = "0.11"
 paste = "1.0"

h2o/kernel/src/cpu/time/chip.rs

@@ -6,10 +6,7 @@ use super::Instant;
 use crate::{cpu::arch::tsc::TSC_CLOCK, dev::hpet::HPET_CLOCK};
 
 pub static CLOCK: Azy<&'static dyn ClockChip> = Azy::new(|| {
-    let ret: &'static dyn ClockChip = match *TSC_CLOCK {
-        Some(ref tsc) => tsc,
-        None => HPET_CLOCK.as_ref().expect("No available clock"),
-    };
+    let ret: &'static dyn ClockChip = &*TSC_CLOCK;
     crate::log::HAS_TIME.store(true, Release);
     ret
 });
@@ -43,7 +40,7 @@ pub fn calibrate(
     let tries = 3;
     let iter_ms = [10u64, 20];
     let mut best = [u64::MAX, u64::MAX];
-    for (i, &duration) in iter_ms.iter().enumerate() {
+    for (best, &duration) in best.iter_mut().zip(iter_ms.iter()) {
         for _ in 0..tries {
             unsafe {
                 CALIB_CLOCK.prepare(duration);
@@ -51,7 +48,6 @@ pub fn calibrate(
 
                 let start = get_start();
                 CALIB_CLOCK.cycle(duration);
-                let best = best.get_unchecked_mut(i);
                 *best = (*best).min(get_end() - start);
 
                 CALIB_CLOCK.cleanup();

h2o/kernel/src/cpu/x86_64/tsc.rs

@@ -6,12 +6,29 @@ use crate::cpu::time::{
     Instant,
 };
 
-pub static TSC_CLOCK: Azy<Option<TscClock>> = Azy::new(TscClock::new);
+pub static TSC_CLOCK: Azy<TscClock> = Azy::new(|| {
+    if CpuId::new()
+        .get_advanced_power_mgmt_info()
+        .map_or(true, |info| !info.has_invariant_tsc())
+    {
+        log::warn!("The TSC is not invariant. Ticks will be unreliable.");
+    }
+
+    let khz = crate::cpu::time::chip::calibrate(|| {}, rdtsc, rdtsc, || {});
+    let initial = rdtsc();
+    let (mul, sft) = factor_from_freq(khz);
+    log::info!("CPU Timestamp frequency: {} KHz", khz);
+    TscClock {
+        initial,
+        mul: mul as u128,
+        sft: sft as u128,
+    }
+});
 
 pub struct TscClock {
-    initial: u64,
-    mul: u128,
-    sft: u128,
+    pub initial: u64,
+    pub mul: u128,
+    pub sft: u128,
 }
 
 impl ClockChip for TscClock {
@@ -21,23 +38,3 @@ impl ClockChip for TscClock {
         unsafe { Instant::from_raw(ns) }
     }
 }
-
-impl TscClock {
-    pub fn new() -> Option<TscClock> {
-        let cpuid = CpuId::new();
-        cpuid
-            .get_advanced_power_mgmt_info()?
-            .has_invariant_tsc()
-            .then(|| {
-                let khz = crate::cpu::time::chip::calibrate(|| {}, rdtsc, rdtsc, || {});
-                let initial = rdtsc();
-                let (mul, sft) = factor_from_freq(khz);
-                log::info!("CPU Timestamp frequency: {} KHz", khz);
-                TscClock {
-                    initial,
-                    mul: mul as u128,
-                    sft: sft as u128,
-                }
-            })
-    }
-}

h2o/kernel/src/mem/space/virt.rs

@@ -120,7 +120,8 @@ impl Virt {
         layout: Layout,
         flags: Flags,
     ) -> Result<LAddr> {
-        if phys == VDSO.1
+        let set_vdso = phys == VDSO.1;
+        if set_vdso
             && (offset.is_some()
                 || phys_offset != 0
                 || layout.size() != VDSO.1.len()
@@ -143,9 +144,10 @@ impl Virt {
         let mut children = self.children.lock();
         let space = self.space.upgrade().ok_or(EKILLED)?;
 
-        let set_vdso = phys == VDSO.1;
         if set_vdso {
-            check_set_vdso(&space.vdso, phys_offset, layout, flags)?;
+            if PREEMPT.scope(|| space.vdso.lock().is_some()) {
+                return Err(EACCES);
+            }
             if self as *const _ != Arc::as_ptr(&space.root) {
                 return Err(EACCES);
             }
@@ -179,7 +181,7 @@ impl Virt {
         let children = self.children.lock();
         let space = self.space.upgrade().ok_or(EKILLED)?;
 
-        let vdso = { *space.vdso.lock() };
+        let vdso = *space.vdso.lock();
         for (&base, child) in children
             .range(..end)
             .take_while(|(&base, child)| start <= child.end(base))
@@ -222,7 +224,7 @@ impl Virt {
         let mut children = self.children.lock();
         let space = self.space.upgrade().ok_or(EKILLED)?;
 
-        let vdso = { *space.vdso.lock() };
+        let vdso = *space.vdso.lock();
         for (&base, child) in children
             .range(..end)
             .take_while(|(&base, child)| start <= child.end(base))
@@ -307,31 +309,6 @@ fn check_layout(layout: Layout) -> Result<Layout> {
     Ok(layout.pad_to_align())
 }
 
-fn check_set_vdso(
-    vdso: &Mutex<Option<LAddr>>,
-    phys_offset: usize,
-    layout: Layout,
-    flags: Flags,
-) -> Result {
-    if PREEMPT.scope(|| vdso.lock().is_some()) {
-        return Err(EACCES);
-    }
-
-    if phys_offset != 0 {
-        return Err(EACCES);
-    }
-
-    if layout.size() != VDSO.1.len() || layout.align() != PAGE_SIZE {
-        return Err(EACCES);
-    }
-
-    if flags != VDSO.0 {
-        return Err(EACCES);
-    }
-
-    Ok(())
-}
-
 fn check_vdso(vdso: Option<LAddr>, base: LAddr, end: LAddr) -> bool {
     let vdso_size = VDSO.1.len();
 

h2o/kernel/src/sched/ipc.rs

@@ -10,7 +10,7 @@ use core::{
 };
 
 use spin::Mutex;
-pub use sv_call::ipc::{SIG_GENERIC, SIG_READ, SIG_WRITE, SIG_TIMER};
+pub use sv_call::ipc::{SIG_GENERIC, SIG_READ, SIG_TIMER, SIG_WRITE};
 
 pub use self::{
     arsc::Arsc,

h2o/kernel/src/sched/task/boot.rs

@@ -8,6 +8,7 @@ use targs::Targs;
 
 use super::{hdl::DefaultFeature, *};
 use crate::{
+    cpu::arch::tsc::TSC_CLOCK,
     mem::space::{Flags, Phys, Virt},
     sched::SCHED,
 };
@@ -49,6 +50,24 @@ fn flags_to_feat(flags: Flags) -> Feature {
 }
 
 pub fn setup() {
+    unsafe {
+        let constants = sv_call::Constants {
+            ticks_offset: TSC_CLOCK.initial,
+            ticks_multiplier: TSC_CLOCK.mul,
+            ticks_shift: TSC_CLOCK.sft,
+        };
+
+        #[allow(clippy::zero_prefixed_literal)]
+        let offset = include!(concat!(
+            env!("CARGO_MANIFEST_DIR"),
+            "/target/constant_offset.rs"
+        ));
+        let ptr = { VDSO.1.base().to_laddr(minfo::ID_OFFSET) }
+            .add(offset)
+            .cast::<sv_call::Constants>();
+        ptr.write(constants);
+    }
+
     let mut objects = hdl::List::new();
 
     // The sequence of kernel objects must match the one defined in

h2o/kernel/syscall/time.json

@@ -2,6 +2,7 @@
     {
         "name": "sv_time_get",
         "returns": "()",
+        "no_call": true,
         "args": [
             {
                 "name": "ptr",

h2o/libs/syscall/src/call.rs

@@ -16,5 +16,24 @@ use crate::{
     Feature, Handle, SerdeReg,
 };
 
+#[cfg(feature = "vdso")]
+#[no_mangle]
+pub unsafe extern "C" fn sv_time_get(ptr: *mut ()) -> crate::c_ty::Status {
+    let ticks = {
+        let (eax, edx): (u32, u32);
+        core::arch::asm!("rdtsc", out("eax")eax, out("edx")edx);
+        ((edx as u64) << 32) | (eax as u64)
+    };
+
+    let c = crate::constants();
+
+    let val = ticks - c.ticks_offset;
+    let ns = (val as u128 * c.ticks_multiplier) >> c.ticks_shift;
+
+    ptr.cast::<u128>().write(ns);
+
+    Status::from_res(Ok(()))
+}
+
 #[cfg(all(not(feature = "stub"), feature = "call"))]
 include!(concat!(env!("CARGO_MANIFEST_DIR"), "/target/call.rs"));

h2o/libs/syscall/src/lib.rs

@@ -1,6 +1,7 @@
 #![no_std]
 #![warn(clippy::missing_panics_doc)]
 #![feature(allocator_api)]
+#![feature(asm_const)]
 #![feature(lang_items)]
 #![feature(linkage)]
 
@@ -26,8 +27,51 @@ pub use self::{
     feat::*,
 };
 
+#[derive(Debug, Copy, Clone)]
+#[repr(C)]
+pub struct Constants {
+    pub ticks_offset: u64,
+    pub ticks_multiplier: u128,
+    pub ticks_shift: u128,
+}
+
+impl Constants {
+    pub const fn new() -> Constants {
+        Constants {
+            ticks_offset: 0,
+            ticks_multiplier: 0,
+            ticks_shift: 0,
+        }
+    }
+}
+
+#[cfg(feature = "vdso")]
+const CONSTANTS_SIZE: usize = core::mem::size_of::<Constants>();
+#[cfg(feature = "vdso")]
+core::arch::global_asm!("
+    .section .rodata
+    .global CONSTANTS
+    .type CONSTANTS, object
+CONSTANTS:
+    .fill {CONSTANTS_SIZE}, 1, 0xcc", 
+    CONSTANTS_SIZE = const CONSTANTS_SIZE
+);
+
+#[cfg(feature = "vdso")]
+fn constants() -> Constants {
+    let mut addr: *const Constants;
+
+    unsafe {
+        core::arch::asm!(
+            "lea {}, [rip + CONSTANTS]",
+            out(reg) addr
+        );
+        core::ptr::read(addr)
+    }
+}
+
 #[cfg(all(not(feature = "call"), feature = "vdso"))]
-compile_error!("The VDSO feature is onlye supported with call feature");
+compile_error!("The VDSO feature is only supported with call feature");
 
 #[cfg(feature = "vdso")]
 #[panic_handler]

h2o/libs/syscall/syscall.ld

@@ -1,9 +1,12 @@
+ENTRY(sv_task_exit)
+
 SECTIONS
 {
     . = SIZEOF_HEADERS;
     .note.gnu.build-id  : { *(.note.gnu.build-id) } :note
 
     .text       : { *(.text*) }     :load
+    .rodata     : { *(.rodata*) }   :load
     .dynamic    : { *(.dynamic*) }  :load :dynamic
 
     /DISCARD/ : { *(.comment*) }

src/lib/elfload/Cargo.toml

@@ -12,5 +12,5 @@ default = ["solvent/default"]
 solvent = {path = "../h2o_rs", default_features = false}
 # External crates
 cstr_core = "0.2"
-goblin = {version = "0.4", default-features = false, features = ["elf32", "elf64", "endian_fd"]}
+goblin = {version = "0.5", default-features = false, features = ["elf32", "elf64", "endian_fd"]}
 log = "0.4"

src/lib/libc/ldso/Cargo.toml

@@ -16,7 +16,7 @@ solvent = {path = "../../h2o_rs"}
 svrt = {path = "../../svrt"}
 # External crates
 cstr_core = "0.2"
-goblin = {version = "0.4", default-features = false, features = ["elf32", "elf64", "endian_fd"]}
+goblin = {version = "0.5", default-features = false, features = ["elf32", "elf64", "endian_fd"]}
 log = "0.4"
 spin = {version = "0.9", features = ["use_ticket_mutex"]}
 

xtask/src/dist.rs

@@ -69,6 +69,8 @@ impl Dist {
             let cd = src_root.join(H2O_SYSCALL);
             let ldscript = cd.join("syscall.ld");
 
+            println!("Building VDSO");
+
             let mut cmd = Command::new(&cargo);
             let cmd = cmd.current_dir(&cd).arg("rustc").args([
                 "--crate-type=cdylib",
@@ -93,19 +95,40 @@ impl Dist {
 
             // Copy the binary to target.
             let bin_dir = Path::new(&target_root).join("x86_64-pc-oceanic/release");
-            fs::copy(
-                bin_dir.join("libsv_call.so"),
-                src_root.join(H2O_KERNEL).join("target/vdso"),
-            )?;
+            let path = src_root.join(H2O_KERNEL).join("target/vdso");
+            fs::copy(bin_dir.join("libsv_call.so"), &path)?;
             Command::new("llvm-ifs")
                 .arg("--input-format=ELF")
                 .arg(format!(
                     "--output-elf={}/sysroot/usr/lib/libh2o.so",
                     target_root
                 ))
-                .arg(src_root.join(H2O_KERNEL).join("target/vdso"))
+                .arg(&path)
                 .status()?
                 .exit_ok()?;
+
+            let out = Command::new("llvm-objdump")
+                .arg("--syms")
+                .arg(&path)
+                .output()?
+                .stdout;
+            let s = String::from_utf8_lossy(&out);
+            let (constants_offset, _) = s
+                .split('\n')
+                .find(|s| s.ends_with("CONSTANTS"))
+                .and_then(|s| s.split_once(' '))
+                .expect("Failed to get CONSTANTS");
+
+            fs::write(
+                src_root.join(H2O_KERNEL).join("target/constant_offset.rs"),
+                format!("0x{}", constants_offset),
+            )?;
+
+            self.gen_debug(
+                "vdso",
+                src_root.join(H2O_KERNEL).join("target"),
+                DEBUG_DIR,
+            )?;
         }
 
         // Build h2o_kernel

xtask/src/gen/syscall.rs

@@ -18,6 +18,8 @@ pub struct SyscallFn {
     pub name: String,
     returns: String,
     args: Vec<SyscallArg>,
+    #[serde(default)]
+    no_call: bool,
 }
 
 fn parse_file(file: impl AsRef<Path>) -> Result<Vec<SyscallFn>, Box<dyn Error>> {
@@ -59,6 +61,10 @@ pub fn gen_rust_calls(funcs: &[SyscallFn], output: impl AsRef<Path>) -> Result<(
     let mut output = BufWriter::new(fs::File::create(output)?);
 
     for (i, func) in funcs.iter().enumerate() {
+        if func.no_call {
+            write!(output, "#[cfg(not(feature = \"vdso\"))] ")?;
+        }
+
         write!(
             output,
             "#[no_mangle] pub unsafe extern \"C\" fn {}(",