diff --git a/README.md b/README.md index baeb034..b97aff6 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,9 @@ and builtin handling live in `initializers.rs` and `builtins.rs`. The native preprocessor keeps shared macro/header state in `src/preprocessor/native.rs`; its `lexer.rs`, `expansion.rs` and `options.rs` child modules own token handling, macro expansion and preprocessing/dependency arguments. +Each backend keeps its generator entry point and core node dispatch in +`src/generator//generator.rs`; `generator/aggregate.rs`, `globals.rs`, +`atomic.rs` and `inline_asm.rs` contain the corresponding target-specific operations. The image workflow runs only when `dockerfiles/dev.Dockerfile`, `.dockerignore`, or `.github/workflows/publish.yaml` changes, or when started manually. Image diff --git a/src/generator/aarch64/generator.rs b/src/generator/aarch64/generator.rs index c2b4536..09dbd62 100644 --- a/src/generator/aarch64/generator.rs +++ b/src/generator/aarch64/generator.rs @@ -1,3 +1,8 @@ +mod aggregate; +mod atomic; +mod globals; +mod inline_asm; + use super::assembly::Assembly; use super::instruction::{ InstOperand::{self, *}, @@ -10,9 +15,7 @@ use crate::error; use crate::func::Func; use crate::generator::Os; use crate::generator::{ScalarLocation, scalar_arguments}; -use crate::global::{GlobalVariable, GlobalVariableData}; -use crate::node::{AtomicOperation, AtomicRmwOperation, Node, NodeType, render_inline_assembly}; -use crate::token::Token; +use crate::node::{Node, NodeType}; use std::{ collections::{HashMap, HashSet}, fmt::Display, @@ -148,264 +151,6 @@ impl<'a> AsmGenerator<'a> { ) } - fn gen_string_literals(&self, string_literals: &[Token]) -> Assembly { - if string_literals.is_empty() { - vec![] - } else { - vec![ - if let Os::MacOS = self.target_os { - ".section __TEXT,__const".into() - } else { - ".section .data".into() - }, - string_literals - .iter() - .enumerate() - .map(|(i, token)| { - let bytes = token.encoded_string(); - vec![ - format!(" .balign {}", token.string_encoding.unit_size()).into(), - format!("L_.str.{}:", i).into(), - format!( - " .byte {}", - bytes - .into_iter() - .map(|byte| byte.to_string()) - .collect::>() - .join(", ") - ) - .into(), - ] - .into() - }) - .collect::>() - .into(), - ] - } - .into() - } - - fn weak_reference(&self, name: &str) -> Assembly { - let directive = match self.target_os { - Os::MacOS => ".weak_reference", - Os::Linux => ".weak", - }; - format!("{directive} {}", self.with_prefix(name)).into() - } - - fn gen_global_variable(&self, name: &str, gv: &GlobalVariable) -> Assembly { - if gv.declaration_only { - return if gv.weak_import { - self.weak_reference(name) - } else { - vec![].into() - }; - } - if gv.common && !gv.internal && !gv.thread_local && gv.data.is_none() { - let alignment = gv.ty.align_of().max(gv.alignment).max(1); - let alignment = match self.target_os { - Os::MacOS => alignment.trailing_zeros() as usize, - Os::Linux => alignment, - }; - return format!( - ".comm {},{},{}", - self.with_prefix(name), - gv.ty.size_of(), - alignment - ) - .into(); - } - vec![ - if let Os::MacOS = self.target_os { - ".section __DATA,__data".into() - } else { - ".section .data".into() - }, - if gv.internal { - vec![].into() - } else { - format!(".globl {}", self.with_prefix(name)).into() - }, - format!(" .balign {}", gv.ty.align_of().max(gv.alignment)).into(), - format!("{}:", self.with_prefix(name)).into(), - self.gen_initializer_element(&gv.ty, gv.data.as_ref()), - ] - .into() - } - - fn gen_initializer_element(&self, ty: &Type, data: Option<&GlobalVariableData>) -> Assembly { - let data = if !matches!(ty, Type::Arr(..) | Type::Struct(_)) { - let mut value = data; - while let Some(GlobalVariableData::Arr(elements)) = value { - value = elements.first(); - } - value - } else { - data - }; - match ty { - Type::Aligned(child, _) => self.gen_initializer_element(child, data), - Type::Struct(struct_type) - if struct_type.has_bit_fields() || struct_type.definition().borrow().is_union => - { - let mut bytes = vec![0; ty.size_of()]; - let mut addresses = std::collections::HashMap::new(); - crate::global::initializer_bytes(ty, data, 0, &mut bytes, &mut addresses); - let mut lines = vec![]; - let mut index = 0; - while index < bytes.len() { - if let Some((symbol, delta)) = addresses.get(&index) { - let symbol = self.address_symbol(symbol); - lines.push(format!(" .8byte {symbol}{delta:+}").into()); - index += 8; - } else { - lines.push(format!(" .byte {}", bytes[index]).into()); - index += 1; - } - } - lines.into() - } - Type::Struct(struct_type) => { - let elements = if let Some(GlobalVariableData::Arr(elements)) = data { - Some(elements) - } else { - None - }; - let mut lines = vec![]; - let mut end = 0; - for (i, member) in struct_type - .members() - .iter() - .take(if struct_type.definition().borrow().is_union { - 1 - } else { - usize::MAX - }) - .enumerate() - { - if member.offset > end { - lines.push(format!(" .zero {}", member.offset - end).into()); - } - lines.push(self.gen_initializer_element( - &member.ty, - elements.and_then(|elements| elements.get(i)), - )); - end = member.offset + member.ty.size_of(); - } - if ty.size_of() > end { - lines.push(format!(" .zero {}", ty.size_of() - end).into()); - } - lines.into() - } - Type::Arr(children_ty, size) => { - let size = size.unwrap_or(0); - if let Some(GlobalVariableData::Arr(v)) = data { - vec![ - v.iter() - .enumerate() - .filter(|(i, _)| *i < size) - .map(|(_, d)| { - self.gen_initializer_element(children_ty.as_ref(), Some(d)) - }) - .collect::>() - .into(), - format!( - " .zero {}", - children_ty.size_of() - * v.iter() - .enumerate() - .filter(|(i, _)| *i < size) - .fold(size, |_, _| { size.saturating_sub(v.len()) }) - ) - .into(), - ] - .into() - } else { - format!(" .zero {}", children_ty.size_of() * size).into() - } - } - Type::Enum(value) => self.gen_initializer_element(&value.underlying.borrow(), data), - Type::I8 | Type::SignedChar => format!( - " .byte {}", - if let Some(GlobalVariableData::Elem(s)) = data { - s - } else { - "0" - } - ) - .into(), - Type::Integer128 { .. } => " .zero 16".into(), - Type::Float16 => " .zero 2".into(), - Type::Vector(..) => format!(" .zero {}", ty.size_of()).into(), - Type::LongDouble(bits) if *bits != 64 => { - let value = if let Some(GlobalVariableData::Elem(value)) = data { - value.parse::().unwrap() - } else { - 0 - }; - format!( - " .8byte {}\n .8byte {}", - value as u64, - (value >> 64) as u64 - ) - .into() - } - Type::Float(_) | Type::LongDouble(_) => format!( - " .{}byte {}", - ty.size_of(), - if let Some(GlobalVariableData::Elem(value)) = data { - value.as_str() - } else { - "0" - } - ) - .into(), - Type::Integer { .. } | Type::LongLong { .. } | Type::Bool => format!( - " {} {}", - if ty.size_of() == 1 { - ".byte".to_string() - } else { - format!(".{}byte", ty.size_of()) - }, - if let Some(GlobalVariableData::Address { symbol, offset }) = data { - format!("{}{offset:+}", self.address_symbol(symbol)) - } else if let Some(GlobalVariableData::Elem(value)) = data { - ty.convert_constant(value.parse().unwrap()).to_string() - } else { - "0".into() - } - ) - .into(), - Type::I32 => format!( - " .4byte {}", - if let Some(GlobalVariableData::Elem(s)) = data { - s - } else { - "0" - } - ) - .into(), - _ => format!( - " .8byte {}", - if let Some(GlobalVariableData::Address { symbol, offset }) = data { - let symbol = self.address_symbol(symbol); - format!("{symbol}{offset:+}") - } else if let Some(GlobalVariableData::Elem(s)) = data { - if s.starts_with("@.str.") { - s.replace('@', "L_") - } else if let Some(name) = s.strip_prefix('@') { - self.address_symbol(name) - } else { - s.clone() - } - } else { - "0".to_string() - } - ) - .into(), - } - } - fn stack_arguments(&self, types: &[Type], fixed: usize) -> (Vec<(usize, usize, usize)>, usize) { let mut offset = 0usize; let mut slots = vec![]; @@ -804,434 +549,6 @@ impl<'a> AsmGenerator<'a> { lines.push(Self::push(X0)); lines.into() } - fn aggregate_load(offset: usize, size: usize) -> Assembly { - let mut lines = vec![Assembly::inst2(MOV, X8, 0)]; - for index in (0..size).rev() { - lines.extend([ - Assembly::inst3(SHL, X8, X8, 8), - Assembly::inst3(ADD, X12, X10, offset + index), - Assembly::inst2(LDRU, X13, Ptr(X12, 1)), - Assembly::inst3(OR, X8, X8, X13), - ]); - } - lines.into() - } - fn aggregate_store(size: usize) -> Assembly { - let mut lines = vec![]; - for index in 0..size { - lines.extend([ - Assembly::inst3(ADD, X12, X11, index), - Assembly::inst2(STR, X8, Ptr(X12, 1)), - ]); - if index + 1 < size { - lines.push(Assembly::inst3(SHR, X8, X8, 8)); - } - } - lines.into() - } - fn aggregate_copy(size: usize) -> Assembly { - (0..size) - .map(|index| { - vec![ - Assembly::inst3(ADD, X12, X10, index), - Assembly::inst2(LDRU, X8, Ptr(X12, 1)), - Assembly::inst3(ADD, X13, X11, index), - Assembly::inst2(STR, X8, Ptr(X13, 1)), - ] - .into() - }) - .collect::>() - .into() - } - fn aggregate_function(&self, name: &str, func: &Func, locals: usize) -> Assembly { - use crate::generator::abi::{self, Location}; - let (types, result, variadic) = func.cty.function_signature().unwrap(); - let signature = abi::signature( - &types, - &result, - crate::generator::Arch::Aarch64, - self.target_os, - types.len(), - variadic, - ); - let offset = locals + if variadic { 192 } else { 128 }; - let mut lines = vec![]; - if !func.internal { - lines.push(format!(".globl {}", self.with_prefix(name)).into()); - } - lines.extend([ - "\t.p2align\t2".into(), - format!("{}:", self.with_prefix(name)).into(), - Assembly::inst3(SUB, SP, SP, offset + 32), - Assembly::inst3(ADD, X10, SP, offset), - Assembly::inst3(STP, X29, X30, PtrAdd(X10, "#0".into())), - Assembly::inst3(STP, X19, X20, PtrAdd(X10, "#16".into())), - Assembly::inst2(MOV, X29, SP), - Assembly::inst2(MOV, X19, SP), - ]); - if signature.indirect_result { - lines.extend([ - Assembly::inst3(ADD, X10, X29, offset - func.return_pointer.unwrap()), - Assembly::inst2(STR, X8, Ptr(X10, 8)), - ]); - } - for (index, register) in ARGS_REG.iter().enumerate() { - lines.extend([ - Assembly::inst3(ADD, X10, X29, index * 8), - Assembly::inst2(STR, *register, Ptr(X10, 8)), - ]); - } - for index in 0..8 { - lines.extend([ - Assembly::inst2(FMOV, X8, Float(index, 64)), - Assembly::inst3(ADD, X10, X29, 64 + index * if variadic { 16 } else { 8 }), - Assembly::inst2(STR, X8, Ptr(X10, 8)), - ]); - } - for (index, argument) in signature.arguments.iter().enumerate() { - let destination = offset - func.args[index].offset.unwrap(); - if let Some(stack) = argument.memory { - lines.extend([ - Assembly::inst3(ADD, X10, X29, offset + 32 + stack), - Assembly::inst3(ADD, X11, X29, destination), - Self::aggregate_copy(types[index].size_of()), - ]); - } else { - for (part, location) in &argument.parts { - let slot = match *location { - Location::General(n) => n * 8, - Location::Floating(n) => 64 + n * if variadic { 16 } else { 8 }, - Location::Stack(stack) => offset + 32 + stack, - }; - lines.extend([ - Assembly::inst3(ADD, X10, X29, slot), - Assembly::inst2(LDRU, X8, Ptr(X10, 8)), - ]); - if argument.indirect { - lines.extend([ - Assembly::inst2(MOV, X10, X8), - Assembly::inst3(ADD, X11, X29, destination), - Self::aggregate_copy(types[index].size_of()), - ]); - } else { - lines.extend([ - Assembly::inst3(ADD, X11, X29, destination + part.offset), - Self::aggregate_store( - (types[index].size_of() - part.offset).min(part.ty.size_of()), - ), - ]); - } - } - } - } - lines.extend([ - self.gen_node( - func.body.as_ref().unwrap(), - Options { - offset, - breakable_branch_num: 0, - }, - ), - Assembly::inst2(MOV, X0, 0), - Self::epilogue(offset), - ]); - lines.into() - } - fn aggregate_call(&self, node: &Node, options: Options) -> Assembly { - use crate::generator::abi::{self, Location}; - let types = node - .args - .iter() - .rev() - .map(|arg| arg.resolve_type().unwrap()) - .collect::>(); - let result = node.resolve_type().unwrap(); - let signature = abi::signature( - &types, - &result, - crate::generator::Arch::Aarch64, - self.target_os, - node.parameter_types.len(), - node.variadic, - ); - let mut lines = vec![]; - if let Some(callee) = &node.lhs { - lines.push(self.gen_node(callee, options)); - } - for arg in &node.args { - lines.push(self.gen_node(arg, options)); - } - lines.extend([ - Assembly::inst3(SUB, X10, X19, signature.stack_size), - Assembly::inst3(AND, X10, X10, "#-16"), - Assembly::inst2(MOV, SP, X10), - ]); - for (index, argument) in signature.arguments.iter().enumerate() { - if let Some(stack) = argument.memory { - lines.extend([ - Assembly::inst2(LDR, X10, PtrAdd(X19, format!("#{}", index * 8))), - Assembly::inst3(ADD, X11, SP, stack), - Self::aggregate_copy(types[index].size_of()), - ]); - } - } - for (index, argument) in signature.arguments.iter().enumerate() { - if argument.memory.is_some() { - continue; - } - for (part, location) in &argument.parts { - if matches!(types[index], Type::Struct(_)) && !argument.indirect { - lines.extend([ - Assembly::inst2(LDR, X10, PtrAdd(X19, format!("#{}", index * 8))), - Self::aggregate_load( - part.offset, - (types[index].size_of() - part.offset).min(part.ty.size_of()), - ), - ]); - } else { - lines.push(Assembly::inst2( - LDR, - X8, - PtrAdd(X19, format!("#{}", index * 8)), - )); - } - lines.push(match *location { - Location::General(register) => Assembly::inst2(MOV, ARGS_REG[register], X8), - Location::Floating(register) => Assembly::inst2( - FMOV, - Float(register, part.ty.float_bits().unwrap()), - if part.ty.float_bits() == Some(32) { - W8 - } else { - X8 - }, - ), - Location::Stack(stack) => vec![ - Assembly::inst3(ADD, X11, SP, stack), - Assembly::inst2( - STR, - X8, - Ptr( - X11, - if matches!(self.target_os, Os::MacOS) - && index < node.parameter_types.len() - { - part.ty.size_of() - } else { - 8 - }, - ), - ), - ] - .into(), - }); - } - } - if signature.indirect_result { - lines.push(Assembly::inst3( - ADD, - X8, - X29, - options.offset - node.offset.unwrap(), - )); - } - if node.lhs.is_some() { - lines.extend([ - Assembly::inst2(LDR, X12, PtrAdd(X19, format!("#{}", node.args.len() * 8))), - Assembly::inst1(BLR, X12), - ]); - } else { - lines.push(Assembly::inst1(BL, self.with_prefix(&node.global_name))); - } - lines.extend([ - Assembly::inst3( - ADD, - X19, - X19, - (node.args.len() + usize::from(node.lhs.is_some())) * 8, - ), - Assembly::inst2(MOV, SP, X29), - ]); - if matches!(result, Type::Struct(_)) { - if !signature.indirect_result { - let (mut gp, mut fp) = (0, 0); - for part in &signature.result { - if let Some(bits) = part.ty.float_bits() { - lines.push(Assembly::inst2( - FMOV, - if bits == 32 { W8 } else { X8 }, - Float(fp, bits), - )); - fp += 1; - } else { - lines.push(Assembly::inst2(MOV, X8, ARGS_REG[gp])); - gp += 1; - } - lines.extend([ - Assembly::inst3( - ADD, - X11, - X29, - options.offset - node.offset.unwrap() + part.offset, - ), - Self::aggregate_store( - (result.size_of() - part.offset).min(part.ty.size_of()), - ), - ]); - } - } - lines.push(Assembly::inst3( - ADD, - X0, - X29, - options.offset - node.offset.unwrap(), - )); - } else if let Some(bits) = result.float_bits() { - lines.push(Assembly::inst2( - FMOV, - if bits == 32 { W0 } else { X0 }, - Float(0, bits), - )); - } else { - lines.push(Self::normalize_integer(&result, X0)); - } - lines.push(Self::push(X0)); - lines.into() - } - fn aggregate_return(&self, node: &Node, options: Options) -> Assembly { - use crate::generator::abi; - let ty = node.resolve_type().unwrap(); - let parts = abi::aggregate_parts(&ty, &crate::generator::Arch::Aarch64); - let mut lines = vec![ - self.gen_node(node.lhs.as_ref().unwrap(), options), - Self::pop(X10), - ]; - if parts.is_empty() { - lines.extend([ - Assembly::inst3(ADD, X11, X29, options.offset - node.offset.unwrap()), - Assembly::inst2(LDR, X11, Ptr(X11, 8)), - Self::aggregate_copy(ty.size_of()), - ]); - } else { - let (mut gp, mut fp) = (0, 0); - for part in parts { - lines.push(Self::aggregate_load( - part.offset, - (ty.size_of() - part.offset).min(part.ty.size_of()), - )); - lines.push(if let Some(bits) = part.ty.float_bits() { - let n = fp; - fp += 1; - Assembly::inst2(FMOV, Float(n, bits), if bits == 32 { W8 } else { X8 }) - } else { - let n = gp; - gp += 1; - Assembly::inst2(MOV, ARGS_REG[n], X8) - }); - } - } - lines.push(Self::epilogue(options.offset)); - lines.into() - } - fn atomic_update(node: &Node, operation: AtomicRmwOperation, return_new: bool) -> Assembly { - let ty = node.args[0].dest_type().unwrap(); - let label = format!(".Latomic.{}", node.token.as_ref().unwrap().pos); - let mut lines = vec![ - format!("{label}:").into(), - Assembly::inst2(LDAXR, X10, Ptr(X8, ty.size_of())), - ]; - lines.push(match operation { - AtomicRmwOperation::Exchange => Assembly::inst2(MOV, X11, X9), - AtomicRmwOperation::Nand => vec![ - Assembly::inst3(AND, X11, X10, X9), - Assembly::inst2(MVN, X11, X11), - ] - .into(), - _ => Assembly::inst3( - match operation { - AtomicRmwOperation::Add => ADD, - AtomicRmwOperation::Sub => SUB, - AtomicRmwOperation::And => AND, - AtomicRmwOperation::Or => OR, - AtomicRmwOperation::Xor => XOR, - _ => unreachable!(), - }, - X11, - X10, - X9, - ), - }); - lines.extend([ - Assembly::inst3(STLXR, W0, X11, Ptr(X8, ty.size_of())), - Assembly::inst2(CMP, X0, 0), - Assembly::inst1(JNE, label), - ]); - let result = if return_new { X11 } else { X10 }; - lines.extend([Self::normalize_integer(&ty, result), Self::push(result)]); - lines.into() - } - - fn atomic_compare_exchange(node: &Node) -> Assembly { - let size = node.args[0].dest_type().unwrap().size_of(); - let prefix = format!(".Latomic.cas.{}", node.token.as_ref().unwrap().pos); - vec![ - Self::pop(X0), - Self::pop(X0), - Self::pop(X0), - Self::pop(X10), - Self::pop(X9), - Self::pop(X8), - Assembly::inst2(LDRU, X12, Ptr(X9, size)), - format!("{prefix}.retry:").into(), - Assembly::inst2(LDAXR, X11, Ptr(X8, size)), - Assembly::inst2(CMP, X11, X12), - Assembly::inst1(JNE, format!("{prefix}.failure")), - Assembly::inst3(STLXR, W0, X10, Ptr(X8, size)), - Assembly::inst2(CMP, X0, 0), - Assembly::inst1(JNE, format!("{prefix}.retry")), - Assembly::inst2(MOV, X0, 1), - Assembly::inst1(JMP, format!("{prefix}.done")), - format!("{prefix}.failure:").into(), - Assembly::inst0(CLREX), - Assembly::inst2(STR, X11, Ptr(X9, size)), - Assembly::inst2(MOV, X0, 0), - format!("{prefix}.done:").into(), - Self::push(X0), - ] - .into() - } - - fn gen_atomic(&self, node: &Node, operation: AtomicOperation, options: Options) -> Assembly { - let mut lines = vec![]; - for argument in &node.args { - lines.push(self.gen_node(argument, options)); - } - if matches!(operation, AtomicOperation::CompareExchange(_)) { - lines.push(Self::atomic_compare_exchange(node)); - return lines.into(); - } - lines.push(Self::pop(X10)); - let ty = node.args[0].dest_type().unwrap(); - if operation == AtomicOperation::Load { - lines.extend([ - Self::pop(X8), - Assembly::inst2(LDAR, X8, Ptr(X8, ty.size_of())), - Self::normalize_integer(&ty, X8), - Self::push(X8), - ]); - } else { - lines.extend([Self::pop(X9), Self::pop(X8)]); - if let AtomicOperation::Update(op, return_new) = operation { - lines.push(Self::atomic_update(node, op, return_new)); - } else { - lines.extend([ - Assembly::inst2(STLR, X9, Ptr(X8, ty.size_of())), - Self::push(0), - ]); - } - } - lines.into() - } fn gen_prefetch(&self, node: &Node, write: bool, locality: u8, options: Options) -> Assembly { let level = [1, 3, 2, 1][locality as usize]; @@ -1249,81 +566,6 @@ impl<'a> AsmGenerator<'a> { .into() } - fn gen_inline_assembly(&self, node: &Node, options: Options) -> Assembly { - let registers = [X8, X9, X10, X11, X12, X13, X14]; - let operands = node.children.iter().chain(&node.args).collect::>(); - let template = render_inline_assembly(&node.dest, |index, modifier| { - let natural = operands.get(index)?.resolve_type()?.size_of(); - let width = crate::generator::asm_register_width( - crate::generator::Arch::Aarch64, - modifier, - natural, - )?; - Some(format!( - "{}{}", - if width == 4 { "w" } else { "x" }, - 8 + index - )) - }) - .unwrap_or_else(|| { - self.error_logger.print_error_position( - node.token.as_ref().unwrap().pos, - "unsupported asm register modifier for AArch64", - ); - unreachable!() - }); - let mut lines = vec![]; - let matching = node - .children - .iter() - .enumerate() - .filter(|(index, _)| node.value.unwrap() & (1 << index) != 0) - .collect::>(); - for (_, input) in &matching { - lines.push(self.gen_node(input, options)); - } - for input in &node.args { - lines.push(self.gen_node(input, options)); - } - for (index, input) in node - .args - .iter() - .enumerate() - .rev() - .map(|(index, input)| (index + node.children.len(), input)) - .chain(matching.into_iter().rev()) - { - let register = registers[index]; - lines.push(Self::pop(register)); - if input.resolve_type().unwrap().size_of() == 4 { - lines.push(Self::normalize_integer(&Type::integer(32, false), register)); - } - } - lines.push(Assembly::inst1( - InlineAsm, - InstOperand::InlineAsmTemplate(template), - )); - // Buffer every result before address generation can overwrite operand registers. - for register in registers[..node.children.len()].iter().rev() { - lines.extend([ - Assembly::inst3(SUB, X19, X19, 8), - Assembly::inst3(AND, X20, X19, "#-16"), - Assembly::inst2(MOV, SP, X20), - Assembly::inst2(STR, *register, Ptr(X19, 8)), - ]); - } - for output in &node.children { - lines.push(self.gen_addr(output, options)); - lines.extend([ - Self::pop(X8), - Self::pop(X13), - Assembly::inst2(STR, X13, Ptr(X8, output.resolve_type().unwrap().size_of())), - ]); - } - lines.push(Self::push(0)); - lines.into() - } - // Keep recursive logical chains out of the large dispatch frame in debug builds. fn gen_node(&self, node: &Node, options: Options) -> Assembly { if node diff --git a/src/generator/aarch64/generator/aggregate.rs b/src/generator/aarch64/generator/aggregate.rs new file mode 100644 index 0000000..17a3eaa --- /dev/null +++ b/src/generator/aarch64/generator/aggregate.rs @@ -0,0 +1,346 @@ +use super::ARGS_REG; +use super::AsmGenerator; +use super::Options; +use crate::ctype::Type; +use crate::func::Func; +use crate::generator::Os; +use crate::generator::aarch64::assembly::Assembly; +use crate::generator::aarch64::instruction::{InstOperand::*, InstOperator::*, Register::*}; +use crate::node::Node; + +impl AsmGenerator<'_> { + pub(super) fn aggregate_load(offset: usize, size: usize) -> Assembly { + let mut lines = vec![Assembly::inst2(MOV, X8, 0)]; + for index in (0..size).rev() { + lines.extend([ + Assembly::inst3(SHL, X8, X8, 8), + Assembly::inst3(ADD, X12, X10, offset + index), + Assembly::inst2(LDRU, X13, Ptr(X12, 1)), + Assembly::inst3(OR, X8, X8, X13), + ]); + } + lines.into() + } + + pub(super) fn aggregate_store(size: usize) -> Assembly { + let mut lines = vec![]; + for index in 0..size { + lines.extend([ + Assembly::inst3(ADD, X12, X11, index), + Assembly::inst2(STR, X8, Ptr(X12, 1)), + ]); + if index + 1 < size { + lines.push(Assembly::inst3(SHR, X8, X8, 8)); + } + } + lines.into() + } + + pub(super) fn aggregate_copy(size: usize) -> Assembly { + (0..size) + .map(|index| { + vec![ + Assembly::inst3(ADD, X12, X10, index), + Assembly::inst2(LDRU, X8, Ptr(X12, 1)), + Assembly::inst3(ADD, X13, X11, index), + Assembly::inst2(STR, X8, Ptr(X13, 1)), + ] + .into() + }) + .collect::>() + .into() + } + + pub(super) fn aggregate_function(&self, name: &str, func: &Func, locals: usize) -> Assembly { + use crate::generator::abi::{self, Location}; + let (types, result, variadic) = func.cty.function_signature().unwrap(); + let signature = abi::signature( + &types, + &result, + crate::generator::Arch::Aarch64, + self.target_os, + types.len(), + variadic, + ); + let offset = locals + if variadic { 192 } else { 128 }; + let mut lines = vec![]; + if !func.internal { + lines.push(format!(".globl {}", self.with_prefix(name)).into()); + } + lines.extend([ + "\t.p2align\t2".into(), + format!("{}:", self.with_prefix(name)).into(), + Assembly::inst3(SUB, SP, SP, offset + 32), + Assembly::inst3(ADD, X10, SP, offset), + Assembly::inst3(STP, X29, X30, PtrAdd(X10, "#0".into())), + Assembly::inst3(STP, X19, X20, PtrAdd(X10, "#16".into())), + Assembly::inst2(MOV, X29, SP), + Assembly::inst2(MOV, X19, SP), + ]); + if signature.indirect_result { + lines.extend([ + Assembly::inst3(ADD, X10, X29, offset - func.return_pointer.unwrap()), + Assembly::inst2(STR, X8, Ptr(X10, 8)), + ]); + } + for (index, register) in ARGS_REG.iter().enumerate() { + lines.extend([ + Assembly::inst3(ADD, X10, X29, index * 8), + Assembly::inst2(STR, *register, Ptr(X10, 8)), + ]); + } + for index in 0..8 { + lines.extend([ + Assembly::inst2(FMOV, X8, Float(index, 64)), + Assembly::inst3(ADD, X10, X29, 64 + index * if variadic { 16 } else { 8 }), + Assembly::inst2(STR, X8, Ptr(X10, 8)), + ]); + } + for (index, argument) in signature.arguments.iter().enumerate() { + let destination = offset - func.args[index].offset.unwrap(); + if let Some(stack) = argument.memory { + lines.extend([ + Assembly::inst3(ADD, X10, X29, offset + 32 + stack), + Assembly::inst3(ADD, X11, X29, destination), + Self::aggregate_copy(types[index].size_of()), + ]); + } else { + for (part, location) in &argument.parts { + let slot = match *location { + Location::General(n) => n * 8, + Location::Floating(n) => 64 + n * if variadic { 16 } else { 8 }, + Location::Stack(stack) => offset + 32 + stack, + }; + lines.extend([ + Assembly::inst3(ADD, X10, X29, slot), + Assembly::inst2(LDRU, X8, Ptr(X10, 8)), + ]); + if argument.indirect { + lines.extend([ + Assembly::inst2(MOV, X10, X8), + Assembly::inst3(ADD, X11, X29, destination), + Self::aggregate_copy(types[index].size_of()), + ]); + } else { + lines.extend([ + Assembly::inst3(ADD, X11, X29, destination + part.offset), + Self::aggregate_store( + (types[index].size_of() - part.offset).min(part.ty.size_of()), + ), + ]); + } + } + } + } + lines.extend([ + self.gen_node( + func.body.as_ref().unwrap(), + Options { + offset, + breakable_branch_num: 0, + }, + ), + Assembly::inst2(MOV, X0, 0), + Self::epilogue(offset), + ]); + lines.into() + } + + pub(super) fn aggregate_call(&self, node: &Node, options: Options) -> Assembly { + use crate::generator::abi::{self, Location}; + let types = node + .args + .iter() + .rev() + .map(|arg| arg.resolve_type().unwrap()) + .collect::>(); + let result = node.resolve_type().unwrap(); + let signature = abi::signature( + &types, + &result, + crate::generator::Arch::Aarch64, + self.target_os, + node.parameter_types.len(), + node.variadic, + ); + let mut lines = vec![]; + if let Some(callee) = &node.lhs { + lines.push(self.gen_node(callee, options)); + } + for arg in &node.args { + lines.push(self.gen_node(arg, options)); + } + lines.extend([ + Assembly::inst3(SUB, X10, X19, signature.stack_size), + Assembly::inst3(AND, X10, X10, "#-16"), + Assembly::inst2(MOV, SP, X10), + ]); + for (index, argument) in signature.arguments.iter().enumerate() { + if let Some(stack) = argument.memory { + lines.extend([ + Assembly::inst2(LDR, X10, PtrAdd(X19, format!("#{}", index * 8))), + Assembly::inst3(ADD, X11, SP, stack), + Self::aggregate_copy(types[index].size_of()), + ]); + } + } + for (index, argument) in signature.arguments.iter().enumerate() { + if argument.memory.is_some() { + continue; + } + for (part, location) in &argument.parts { + if matches!(types[index], Type::Struct(_)) && !argument.indirect { + lines.extend([ + Assembly::inst2(LDR, X10, PtrAdd(X19, format!("#{}", index * 8))), + Self::aggregate_load( + part.offset, + (types[index].size_of() - part.offset).min(part.ty.size_of()), + ), + ]); + } else { + lines.push(Assembly::inst2( + LDR, + X8, + PtrAdd(X19, format!("#{}", index * 8)), + )); + } + lines.push(match *location { + Location::General(register) => Assembly::inst2(MOV, ARGS_REG[register], X8), + Location::Floating(register) => Assembly::inst2( + FMOV, + Float(register, part.ty.float_bits().unwrap()), + if part.ty.float_bits() == Some(32) { + W8 + } else { + X8 + }, + ), + Location::Stack(stack) => vec![ + Assembly::inst3(ADD, X11, SP, stack), + Assembly::inst2( + STR, + X8, + Ptr( + X11, + if matches!(self.target_os, Os::MacOS) + && index < node.parameter_types.len() + { + part.ty.size_of() + } else { + 8 + }, + ), + ), + ] + .into(), + }); + } + } + if signature.indirect_result { + lines.push(Assembly::inst3( + ADD, + X8, + X29, + options.offset - node.offset.unwrap(), + )); + } + if node.lhs.is_some() { + lines.extend([ + Assembly::inst2(LDR, X12, PtrAdd(X19, format!("#{}", node.args.len() * 8))), + Assembly::inst1(BLR, X12), + ]); + } else { + lines.push(Assembly::inst1(BL, self.with_prefix(&node.global_name))); + } + lines.extend([ + Assembly::inst3( + ADD, + X19, + X19, + (node.args.len() + usize::from(node.lhs.is_some())) * 8, + ), + Assembly::inst2(MOV, SP, X29), + ]); + if matches!(result, Type::Struct(_)) { + if !signature.indirect_result { + let (mut gp, mut fp) = (0, 0); + for part in &signature.result { + if let Some(bits) = part.ty.float_bits() { + lines.push(Assembly::inst2( + FMOV, + if bits == 32 { W8 } else { X8 }, + Float(fp, bits), + )); + fp += 1; + } else { + lines.push(Assembly::inst2(MOV, X8, ARGS_REG[gp])); + gp += 1; + } + lines.extend([ + Assembly::inst3( + ADD, + X11, + X29, + options.offset - node.offset.unwrap() + part.offset, + ), + Self::aggregate_store( + (result.size_of() - part.offset).min(part.ty.size_of()), + ), + ]); + } + } + lines.push(Assembly::inst3( + ADD, + X0, + X29, + options.offset - node.offset.unwrap(), + )); + } else if let Some(bits) = result.float_bits() { + lines.push(Assembly::inst2( + FMOV, + if bits == 32 { W0 } else { X0 }, + Float(0, bits), + )); + } else { + lines.push(Self::normalize_integer(&result, X0)); + } + lines.push(Self::push(X0)); + lines.into() + } + + pub(super) fn aggregate_return(&self, node: &Node, options: Options) -> Assembly { + use crate::generator::abi; + let ty = node.resolve_type().unwrap(); + let parts = abi::aggregate_parts(&ty, &crate::generator::Arch::Aarch64); + let mut lines = vec![ + self.gen_node(node.lhs.as_ref().unwrap(), options), + Self::pop(X10), + ]; + if parts.is_empty() { + lines.extend([ + Assembly::inst3(ADD, X11, X29, options.offset - node.offset.unwrap()), + Assembly::inst2(LDR, X11, Ptr(X11, 8)), + Self::aggregate_copy(ty.size_of()), + ]); + } else { + let (mut gp, mut fp) = (0, 0); + for part in parts { + lines.push(Self::aggregate_load( + part.offset, + (ty.size_of() - part.offset).min(part.ty.size_of()), + )); + lines.push(if let Some(bits) = part.ty.float_bits() { + let n = fp; + fp += 1; + Assembly::inst2(FMOV, Float(n, bits), if bits == 32 { W8 } else { X8 }) + } else { + let n = gp; + gp += 1; + Assembly::inst2(MOV, ARGS_REG[n], X8) + }); + } + } + lines.push(Self::epilogue(options.offset)); + lines.into() + } +} diff --git a/src/generator/aarch64/generator/atomic.rs b/src/generator/aarch64/generator/atomic.rs new file mode 100644 index 0000000..e4b1f1f --- /dev/null +++ b/src/generator/aarch64/generator/atomic.rs @@ -0,0 +1,116 @@ +use super::AsmGenerator; +use super::Options; +use crate::generator::aarch64::assembly::Assembly; +use crate::generator::aarch64::instruction::{InstOperand::*, InstOperator::*, Register::*}; +use crate::node::{AtomicOperation, AtomicRmwOperation, Node}; + +impl AsmGenerator<'_> { + pub(super) fn atomic_update( + node: &Node, + operation: AtomicRmwOperation, + return_new: bool, + ) -> Assembly { + let ty = node.args[0].dest_type().unwrap(); + let label = format!(".Latomic.{}", node.token.as_ref().unwrap().pos); + let mut lines = vec![ + format!("{label}:").into(), + Assembly::inst2(LDAXR, X10, Ptr(X8, ty.size_of())), + ]; + lines.push(match operation { + AtomicRmwOperation::Exchange => Assembly::inst2(MOV, X11, X9), + AtomicRmwOperation::Nand => vec![ + Assembly::inst3(AND, X11, X10, X9), + Assembly::inst2(MVN, X11, X11), + ] + .into(), + _ => Assembly::inst3( + match operation { + AtomicRmwOperation::Add => ADD, + AtomicRmwOperation::Sub => SUB, + AtomicRmwOperation::And => AND, + AtomicRmwOperation::Or => OR, + AtomicRmwOperation::Xor => XOR, + _ => unreachable!(), + }, + X11, + X10, + X9, + ), + }); + lines.extend([ + Assembly::inst3(STLXR, W0, X11, Ptr(X8, ty.size_of())), + Assembly::inst2(CMP, X0, 0), + Assembly::inst1(JNE, label), + ]); + let result = if return_new { X11 } else { X10 }; + lines.extend([Self::normalize_integer(&ty, result), Self::push(result)]); + lines.into() + } + + pub(super) fn atomic_compare_exchange(node: &Node) -> Assembly { + let size = node.args[0].dest_type().unwrap().size_of(); + let prefix = format!(".Latomic.cas.{}", node.token.as_ref().unwrap().pos); + vec![ + Self::pop(X0), + Self::pop(X0), + Self::pop(X0), + Self::pop(X10), + Self::pop(X9), + Self::pop(X8), + Assembly::inst2(LDRU, X12, Ptr(X9, size)), + format!("{prefix}.retry:").into(), + Assembly::inst2(LDAXR, X11, Ptr(X8, size)), + Assembly::inst2(CMP, X11, X12), + Assembly::inst1(JNE, format!("{prefix}.failure")), + Assembly::inst3(STLXR, W0, X10, Ptr(X8, size)), + Assembly::inst2(CMP, X0, 0), + Assembly::inst1(JNE, format!("{prefix}.retry")), + Assembly::inst2(MOV, X0, 1), + Assembly::inst1(JMP, format!("{prefix}.done")), + format!("{prefix}.failure:").into(), + Assembly::inst0(CLREX), + Assembly::inst2(STR, X11, Ptr(X9, size)), + Assembly::inst2(MOV, X0, 0), + format!("{prefix}.done:").into(), + Self::push(X0), + ] + .into() + } + + pub(super) fn gen_atomic( + &self, + node: &Node, + operation: AtomicOperation, + options: Options, + ) -> Assembly { + let mut lines = vec![]; + for argument in &node.args { + lines.push(self.gen_node(argument, options)); + } + if matches!(operation, AtomicOperation::CompareExchange(_)) { + lines.push(Self::atomic_compare_exchange(node)); + return lines.into(); + } + lines.push(Self::pop(X10)); + let ty = node.args[0].dest_type().unwrap(); + if operation == AtomicOperation::Load { + lines.extend([ + Self::pop(X8), + Assembly::inst2(LDAR, X8, Ptr(X8, ty.size_of())), + Self::normalize_integer(&ty, X8), + Self::push(X8), + ]); + } else { + lines.extend([Self::pop(X9), Self::pop(X8)]); + if let AtomicOperation::Update(op, return_new) = operation { + lines.push(Self::atomic_update(node, op, return_new)); + } else { + lines.extend([ + Assembly::inst2(STLR, X9, Ptr(X8, ty.size_of())), + Self::push(0), + ]); + } + } + lines.into() + } +} diff --git a/src/generator/aarch64/generator/globals.rs b/src/generator/aarch64/generator/globals.rs new file mode 100644 index 0000000..54d91bc --- /dev/null +++ b/src/generator/aarch64/generator/globals.rs @@ -0,0 +1,270 @@ +use super::AsmGenerator; +use crate::ctype::Type; +use crate::generator::Os; +use crate::generator::aarch64::assembly::Assembly; +use crate::global::{GlobalVariable, GlobalVariableData}; +use crate::token::Token; + +impl AsmGenerator<'_> { + pub(super) fn gen_string_literals(&self, string_literals: &[Token]) -> Assembly { + if string_literals.is_empty() { + vec![] + } else { + vec![ + if let Os::MacOS = self.target_os { + ".section __TEXT,__const".into() + } else { + ".section .data".into() + }, + string_literals + .iter() + .enumerate() + .map(|(i, token)| { + let bytes = token.encoded_string(); + vec![ + format!(" .balign {}", token.string_encoding.unit_size()).into(), + format!("L_.str.{}:", i).into(), + format!( + " .byte {}", + bytes + .into_iter() + .map(|byte| byte.to_string()) + .collect::>() + .join(", ") + ) + .into(), + ] + .into() + }) + .collect::>() + .into(), + ] + } + .into() + } + + pub(super) fn weak_reference(&self, name: &str) -> Assembly { + let directive = match self.target_os { + Os::MacOS => ".weak_reference", + Os::Linux => ".weak", + }; + format!("{directive} {}", self.with_prefix(name)).into() + } + + pub(super) fn gen_global_variable(&self, name: &str, gv: &GlobalVariable) -> Assembly { + if gv.declaration_only { + return if gv.weak_import { + self.weak_reference(name) + } else { + vec![].into() + }; + } + if gv.common && !gv.internal && !gv.thread_local && gv.data.is_none() { + let alignment = gv.ty.align_of().max(gv.alignment).max(1); + let alignment = match self.target_os { + Os::MacOS => alignment.trailing_zeros() as usize, + Os::Linux => alignment, + }; + return format!( + ".comm {},{},{}", + self.with_prefix(name), + gv.ty.size_of(), + alignment + ) + .into(); + } + vec![ + if let Os::MacOS = self.target_os { + ".section __DATA,__data".into() + } else { + ".section .data".into() + }, + if gv.internal { + vec![].into() + } else { + format!(".globl {}", self.with_prefix(name)).into() + }, + format!(" .balign {}", gv.ty.align_of().max(gv.alignment)).into(), + format!("{}:", self.with_prefix(name)).into(), + self.gen_initializer_element(&gv.ty, gv.data.as_ref()), + ] + .into() + } + + pub(super) fn gen_initializer_element( + &self, + ty: &Type, + data: Option<&GlobalVariableData>, + ) -> Assembly { + let data = if !matches!(ty, Type::Arr(..) | Type::Struct(_)) { + let mut value = data; + while let Some(GlobalVariableData::Arr(elements)) = value { + value = elements.first(); + } + value + } else { + data + }; + match ty { + Type::Aligned(child, _) => self.gen_initializer_element(child, data), + Type::Struct(struct_type) + if struct_type.has_bit_fields() || struct_type.definition().borrow().is_union => + { + let mut bytes = vec![0; ty.size_of()]; + let mut addresses = std::collections::HashMap::new(); + crate::global::initializer_bytes(ty, data, 0, &mut bytes, &mut addresses); + let mut lines = vec![]; + let mut index = 0; + while index < bytes.len() { + if let Some((symbol, delta)) = addresses.get(&index) { + let symbol = self.address_symbol(symbol); + lines.push(format!(" .8byte {symbol}{delta:+}").into()); + index += 8; + } else { + lines.push(format!(" .byte {}", bytes[index]).into()); + index += 1; + } + } + lines.into() + } + Type::Struct(struct_type) => { + let elements = if let Some(GlobalVariableData::Arr(elements)) = data { + Some(elements) + } else { + None + }; + let mut lines = vec![]; + let mut end = 0; + for (i, member) in struct_type + .members() + .iter() + .take(if struct_type.definition().borrow().is_union { + 1 + } else { + usize::MAX + }) + .enumerate() + { + if member.offset > end { + lines.push(format!(" .zero {}", member.offset - end).into()); + } + lines.push(self.gen_initializer_element( + &member.ty, + elements.and_then(|elements| elements.get(i)), + )); + end = member.offset + member.ty.size_of(); + } + if ty.size_of() > end { + lines.push(format!(" .zero {}", ty.size_of() - end).into()); + } + lines.into() + } + Type::Arr(children_ty, size) => { + let size = size.unwrap_or(0); + if let Some(GlobalVariableData::Arr(v)) = data { + vec![ + v.iter() + .enumerate() + .filter(|(i, _)| *i < size) + .map(|(_, d)| { + self.gen_initializer_element(children_ty.as_ref(), Some(d)) + }) + .collect::>() + .into(), + format!( + " .zero {}", + children_ty.size_of() + * v.iter() + .enumerate() + .filter(|(i, _)| *i < size) + .fold(size, |_, _| { size.saturating_sub(v.len()) }) + ) + .into(), + ] + .into() + } else { + format!(" .zero {}", children_ty.size_of() * size).into() + } + } + Type::Enum(value) => self.gen_initializer_element(&value.underlying.borrow(), data), + Type::I8 | Type::SignedChar => format!( + " .byte {}", + if let Some(GlobalVariableData::Elem(s)) = data { + s + } else { + "0" + } + ) + .into(), + Type::Integer128 { .. } => " .zero 16".into(), + Type::Float16 => " .zero 2".into(), + Type::Vector(..) => format!(" .zero {}", ty.size_of()).into(), + Type::LongDouble(bits) if *bits != 64 => { + let value = if let Some(GlobalVariableData::Elem(value)) = data { + value.parse::().unwrap() + } else { + 0 + }; + format!( + " .8byte {}\n .8byte {}", + value as u64, + (value >> 64) as u64 + ) + .into() + } + Type::Float(_) | Type::LongDouble(_) => format!( + " .{}byte {}", + ty.size_of(), + if let Some(GlobalVariableData::Elem(value)) = data { + value.as_str() + } else { + "0" + } + ) + .into(), + Type::Integer { .. } | Type::LongLong { .. } | Type::Bool => format!( + " {} {}", + if ty.size_of() == 1 { + ".byte".to_string() + } else { + format!(".{}byte", ty.size_of()) + }, + if let Some(GlobalVariableData::Address { symbol, offset }) = data { + format!("{}{offset:+}", self.address_symbol(symbol)) + } else if let Some(GlobalVariableData::Elem(value)) = data { + ty.convert_constant(value.parse().unwrap()).to_string() + } else { + "0".into() + } + ) + .into(), + Type::I32 => format!( + " .4byte {}", + if let Some(GlobalVariableData::Elem(s)) = data { + s + } else { + "0" + } + ) + .into(), + _ => format!( + " .8byte {}", + if let Some(GlobalVariableData::Address { symbol, offset }) = data { + let symbol = self.address_symbol(symbol); + format!("{symbol}{offset:+}") + } else if let Some(GlobalVariableData::Elem(s)) = data { + if s.starts_with("@.str.") { + s.replace('@', "L_") + } else if let Some(name) = s.strip_prefix('@') { + self.address_symbol(name) + } else { + s.clone() + } + } else { + "0".to_string() + } + ) + .into(), + } + } +} diff --git a/src/generator/aarch64/generator/inline_asm.rs b/src/generator/aarch64/generator/inline_asm.rs new file mode 100644 index 0000000..9842ce6 --- /dev/null +++ b/src/generator/aarch64/generator/inline_asm.rs @@ -0,0 +1,87 @@ +use super::AsmGenerator; +use super::Options; +use crate::ctype::Type; +use crate::generator::aarch64::assembly::Assembly; +use crate::generator::aarch64::instruction::{ + InstOperand::{self, *}, + InstOperator::*, + Register::*, +}; +use crate::node::{Node, render_inline_assembly}; + +impl AsmGenerator<'_> { + pub(super) fn gen_inline_assembly(&self, node: &Node, options: Options) -> Assembly { + let registers = [X8, X9, X10, X11, X12, X13, X14]; + let operands = node.children.iter().chain(&node.args).collect::>(); + let template = render_inline_assembly(&node.dest, |index, modifier| { + let natural = operands.get(index)?.resolve_type()?.size_of(); + let width = crate::generator::asm_register_width( + crate::generator::Arch::Aarch64, + modifier, + natural, + )?; + Some(format!( + "{}{}", + if width == 4 { "w" } else { "x" }, + 8 + index + )) + }) + .unwrap_or_else(|| { + self.error_logger.print_error_position( + node.token.as_ref().unwrap().pos, + "unsupported asm register modifier for AArch64", + ); + unreachable!() + }); + let mut lines = vec![]; + let matching = node + .children + .iter() + .enumerate() + .filter(|(index, _)| node.value.unwrap() & (1 << index) != 0) + .collect::>(); + for (_, input) in &matching { + lines.push(self.gen_node(input, options)); + } + for input in &node.args { + lines.push(self.gen_node(input, options)); + } + for (index, input) in node + .args + .iter() + .enumerate() + .rev() + .map(|(index, input)| (index + node.children.len(), input)) + .chain(matching.into_iter().rev()) + { + let register = registers[index]; + lines.push(Self::pop(register)); + if input.resolve_type().unwrap().size_of() == 4 { + lines.push(Self::normalize_integer(&Type::integer(32, false), register)); + } + } + lines.push(Assembly::inst1( + InlineAsm, + InstOperand::InlineAsmTemplate(template), + )); + // Buffer every result before address generation can overwrite operand registers. + for register in registers[..node.children.len()].iter().rev() { + lines.extend([ + Assembly::inst3(SUB, X19, X19, 8), + Assembly::inst3(AND, X20, X19, "#-16"), + Assembly::inst2(MOV, SP, X20), + Assembly::inst2(STR, *register, Ptr(X19, 8)), + ]); + } + for output in &node.children { + lines.push(self.gen_addr(output, options)); + lines.extend([ + Self::pop(X8), + Self::pop(X13), + Assembly::inst2(STR, X13, Ptr(X8, output.resolve_type().unwrap().size_of())), + ]); + } + lines.push(Self::push(0)); + lines.into() + } +} diff --git a/src/generator/llvm/generator.rs b/src/generator/llvm/generator.rs index 5ec192a..d6d1a71 100644 --- a/src/generator/llvm/generator.rs +++ b/src/generator/llvm/generator.rs @@ -1,3 +1,8 @@ +mod aggregate; +mod atomic; +mod globals; +mod inline_asm; + use std::{borrow::Cow, collections::HashMap, vec}; use crate::{ @@ -5,9 +10,7 @@ use crate::{ ctype::Type, error, func::Func, - global::{GlobalVariable, GlobalVariableData}, - node::{AtomicOperation, AtomicRmwOperation, Node, NodeType, render_inline_assembly}, - token::Token, + node::{Node, NodeType}, }; #[derive(Debug)] @@ -149,155 +152,10 @@ impl<'a> IrGenerator<'a> { } } - fn gen_string_literals(&self, string_literals: &[Token]) -> Vec { - string_literals - .iter() - .enumerate() - .map(|(i, token)| { - let bytes = token.encoded_string(); - let width = token.string_encoding.unit_size(); - let element_type = token.string_encoding.element_type(); - let initializer = if width == 1 { - format!( - "c\"{}\"", - bytes.iter().map(|byte| format!("\\{:02X}", byte)).collect::() - ) - } else { - let elements = bytes.chunks_exact(width).map(|chunk| { - let mut value = [0u8; 4]; - value[..width].copy_from_slice(chunk); - format!("{element_type} {}", u32::from_le_bytes(value)) - }).collect::>().join(", "); - format!("[{elements}]") - }; - format!( - "@.str.{i} = private unnamed_addr constant [{} x {element_type}] {initializer}, align {width}", - bytes.len() / width, - ) - }) - .collect::>() - } - fn align(ty: &Type) -> usize { ty.align_of() } - fn gen_global_variable(&self, name: &str, gv: &GlobalVariable) -> String { - if gv.declaration_only { - // Opaque pointers can reference an external record before its layout is known. - let incomplete_record = matches!(gv.ty.unqualified(), Type::Struct(record) if !record.definition().borrow().complete); - let ty = if incomplete_record { - "i8".into() - } else { - Self::gen_type(&gv.ty) - }; - return format!( - "{} = {} {}{}global {}, align {}", - self.symbol_name(name), - if gv.weak_import { - "extern_weak" - } else { - "external" - }, - if gv.hidden && !gv.internal { - "hidden " - } else { - "" - }, - if gv.thread_local { "thread_local " } else { "" }, - ty, - Self::align(&gv.ty).max(gv.alignment).max(1) - ); - } - format!( - "{} = {}{}{}global {}, align {}", - self.symbol_name(name), - if gv.internal { - "internal " - } else if gv.common && !gv.thread_local && gv.data.is_none() { - "common " - } else { - "" - }, - if gv.hidden && !gv.internal { - "hidden " - } else { - "" - }, - if gv.thread_local { "thread_local " } else { "" }, - if Self::contains_packed_record(&gv.ty) { - self.packed_global_initializer(&gv.ty, gv.data.as_ref()) - } else { - self.gen_initializer_element(&gv.ty, gv.data.as_ref()) - }, - Self::align(&gv.ty).max(gv.alignment), - ) - } - - fn contains_packed_record(ty: &Type) -> bool { - match ty.unqualified() { - Type::Struct(record) => { - record.definition().borrow().packing.is_some() - || ty.contains_typedef_alignment() - || record - .members() - .iter() - .any(|member| Self::contains_packed_record(&member.ty)) - } - Type::Arr(child, _) => Self::contains_packed_record(child), - _ => false, - } - } - - fn packed_global_initializer(&self, ty: &Type, data: Option<&GlobalVariableData>) -> String { - let mut bytes = vec![0; ty.size_of()]; - let mut addresses = HashMap::new(); - crate::global::initializer_bytes(ty, data, 0, &mut bytes, &mut addresses); - let mut addresses = addresses.into_iter().collect::>(); - addresses.sort_by_key(|(offset, _)| *offset); - let mut types = vec![]; - let mut values = vec![]; - let mut cursor = 0; - for (offset, (symbol, delta)) in addresses { - if offset > cursor { - let count = offset - cursor; - types.push(format!("[{count} x i8]")); - values.push(format!( - "[{count} x i8] c\"{}\"", - bytes[cursor..offset] - .iter() - .map(|byte| format!("\\{byte:02X}")) - .collect::() - )); - } - types.push("ptr".into()); - let pointer = if delta == 0 { - self.symbol_name(&symbol) - } else { - format!( - "getelementptr (i8, ptr {}, i64 {delta})", - self.symbol_name(&symbol) - ) - }; - values.push(format!("ptr {pointer}")); - cursor = offset + 8; - } - if cursor < bytes.len() { - let count = bytes.len() - cursor; - types.push(format!("[{count} x i8]")); - values.push(format!( - "[{count} x i8] c\"{}\"", - bytes[cursor..] - .iter() - .map(|byte| format!("\\{byte:02X}")) - .collect::() - )); - } - // Opaque pointers allow the definition to use an exact packed storage - // type, including relocations at offsets below pointer alignment. - format!("<{{ {} }}> <{{ {} }}>", types.join(", "), values.join(", ")) - } - fn access_alignment(node: &Node) -> usize { if matches!(node.nt, NodeType::Deref) { 1 @@ -452,179 +310,6 @@ impl<'a> IrGenerator<'a> { Self::gen_type(ty) } - fn gen_initializer_element(&self, ty: &Type, data: Option<&GlobalVariableData>) -> String { - let data = if !matches!(ty, Type::Arr(..) | Type::Struct(_)) { - let mut value = data; - while let Some(GlobalVariableData::Arr(elements)) = value { - value = elements.first(); - } - value - } else { - data - }; - if ty.integer_info().is_some_and(|(bits, _)| bits == 64) { - if let Some(GlobalVariableData::Address { symbol, offset }) = data { - let pointer = if *offset == 0 { - self.symbol_name(symbol) - } else { - format!( - "getelementptr (i8, ptr {}, i64 {offset})", - self.symbol_name(symbol) - ) - }; - return format!("i64 ptrtoint (ptr {pointer} to i64)"); - } - } - match ty { - Type::Aligned(child, _) => self.gen_initializer_element(child, data), - Type::Vla(..) => unreachable!("variable length arrays have automatic storage"), - Type::Integer128 { .. } => "<2 x i64> zeroinitializer".into(), - Type::Float16 => "half zeroinitializer".into(), - Type::Vector(..) => format!("{} zeroinitializer", Self::gen_type(ty)), - Type::LongDouble(80) => { - let bits = if let Some(GlobalVariableData::Elem(value)) = data { - value.parse::().unwrap() - } else { - 0 - }; - format!("x86_fp80 {}", crate::floating::llvm_literal(bits)) - } - Type::LongDouble(128) => { - format!("{} zeroinitializer", Self::gen_type(ty)) - } - Type::Float(_) | Type::LongDouble(_) => { - let bits = if let Some(GlobalVariableData::Elem(value)) = data { - value.parse::().unwrap() as usize - } else { - 0 - }; - format!("{} {}", Self::gen_type(ty), Self::float_literal(ty, bits)) - } - - Type::Struct(struct_type) - if struct_type.definition().borrow().is_union || struct_type.has_bit_fields() => - { - let mut bytes = vec![0u8; ty.size_of()]; - let mut addresses = HashMap::new(); - crate::global::initializer_bytes(ty, data, 0, &mut bytes, &mut addresses); - let width = ty.align_of().min(8); - let elements = bytes - .chunks(width) - .enumerate() - .map(|(index, chunk)| { - let mut word = [0u8; 8]; - word[..chunk.len()].copy_from_slice(chunk); - if let Some((symbol, offset)) = addresses.get(&(index * width)) { - let pointer = if *offset == 0 { - self.symbol_name(symbol) - } else { - format!( - "getelementptr (i8, ptr {}, i64 {offset})", - self.symbol_name(symbol) - ) - }; - format!("i64 ptrtoint (ptr {pointer} to i64)") - } else { - format!("i{} {}", width * 8, u64::from_le_bytes(word)) - } - }) - .collect::>() - .join(", "); - if ty.align_of() > width { - format!( - "{} {{ [{} x i{}] [{}], [0 x {}] zeroinitializer }}", - Self::gen_type(ty), - ty.size_of() / width, - width * 8, - elements, - Self::alignment_type(ty.align_of()), - ) - } else { - format!("{} [{}]", Self::gen_type(ty), elements) - } - } - Type::Qualified(child, _) => self.gen_initializer_element(child, data), - Type::Struct(struct_type) => { - if let Some(GlobalVariableData::Arr(elements)) = data { - let mut fields = vec![]; - let mut offset = 0usize; - for (index, member) in struct_type.members().iter().enumerate() { - let aligned_offset = - offset.div_ceil(member.ty.align_of()) * member.ty.align_of(); - if member.offset > aligned_offset { - let padding = format!("[{} x i8]", member.offset - aligned_offset); - fields.push(format!("{padding} zeroinitializer")); - } - fields.push(self.gen_initializer_element(&member.ty, elements.get(index))); - offset = member.offset + member.ty.size_of(); - } - if ty.size_of() > offset { - let padding = format!("[{} x i8]", ty.size_of() - offset); - fields.push(format!("{padding} zeroinitializer")); - } - if let Some(anchor) = Self::record_alignment_member(struct_type) { - fields.push(format!("{anchor} zeroinitializer")); - } - format!("{} {{ {} }}", Self::gen_type(ty), fields.join(", ")) - } else { - format!("{} zeroinitializer", Self::gen_type(ty)) - } - } - Type::Arr(children_ty, size) => { - if let Some(GlobalVariableData::Arr(v)) = data { - format!( - "{} [{}]", - Self::gen_type(ty), - (0..size.unwrap_or(0)) - .map(|i| self.gen_initializer_element(children_ty, v.get(i))) - .collect::>() - .join(", "), - ) - } else { - format!("{} zeroinitializer", Self::gen_type(ty)) - } - } - Type::Ptr(_) => format!( - "ptr {}", - match data { - Some(GlobalVariableData::Address { symbol, offset }) => { - if *offset == 0 { - self.symbol_name(symbol) - } else { - format!( - "getelementptr (i8, ptr {}, i64 {offset})", - self.symbol_name(symbol) - ) - } - } - Some(GlobalVariableData::Elem(value)) if value.starts_with('@') => - value.clone(), - Some(GlobalVariableData::Elem(value)) if value != "0" => - format!("inttoptr (i64 {} to ptr)", value), - _ => "null".to_string(), - } - ), - Type::Enum(value) => self.gen_initializer_element(&value.underlying.borrow(), data), - Type::I8 - | Type::SignedChar - | Type::I32 - | Type::LongLong { .. } - | Type::Integer { .. } - | Type::Bool => format!( - "{} {}", - Self::gen_type(ty), - if let Some(GlobalVariableData::Elem(s)) = data { - s - } else { - "0" - } - ), - Type::Void | Type::Func(_, _) | Type::VariadicFunc(_, _) => { - format!("{} zeroinitializer", Self::gen_type(ty)) - } - } - } - fn target_arch() -> crate::generator::Arch { if cfg!(target_arch = "aarch64") { crate::generator::Arch::Aarch64 @@ -639,565 +324,7 @@ impl<'a> IrGenerator<'a> { crate::generator::Os::Linux } } - fn aggregate_result_type(ty: &Type) -> String { - let parts = crate::generator::abi::aggregate_parts(ty, &Self::target_arch()); - if parts.is_empty() { - "void".into() - } else if parts.len() == 1 { - Self::gen_type(&parts[0].ty) - } else { - format!( - "{{ {} }}", - parts - .iter() - .map(|part| Self::gen_type(&part.ty)) - .collect::>() - .join(", ") - ) - } - } - fn aggregate_signature( - types: &[impl std::borrow::Borrow], - result: &Type, - fixed: usize, - variadic: bool, - ) -> crate::generator::abi::Signature { - use crate::generator::abi; - let mut signature = abi::signature( - types, - result, - Self::target_arch(), - Self::target_os(), - fixed, - variadic, - ); - if cfg!(all(target_arch = "aarch64", target_os = "macos")) { - for (ty, argument) in types.iter().zip(&mut signature.arguments) { - if let Some(offset) = argument.memory { - let parts = - abi::aggregate_parts(std::borrow::Borrow::borrow(ty), &Self::target_arch()); - if !parts.is_empty() { - // AArch64 lowers direct aggregates using the same coercion - // whether they occupy registers or stack slots. LLVM byval - // has different stack-slot accounting on Darwin. - argument.parts = parts - .into_iter() - .map(|part| (part, abi::Location::Stack(offset))) - .collect(); - argument.memory = None; - } - } - } - } - signature - } - fn aggregate_parameter_types( - ty: &Type, - argument: &crate::generator::abi::Argument, - ) -> Vec { - if argument.memory.is_some() { - return vec![format!( - "ptr byval({}) align {}", - Self::gen_type(ty), - argument.stack_alignment - )]; - } - if argument.indirect { - return vec!["ptr".into()]; - } - if cfg!(target_arch = "aarch64") - && matches!(ty.unqualified(), Type::Struct(_)) - && (if cfg!(target_os = "linux") { - ty.natural_align_of() - } else { - ty.align_of() - }) >= 16 - && argument.parts.len() == 2 - && argument - .parts - .iter() - .all(|(part, _)| part.ty.float_bits().is_none()) - { - return vec!["i128".into()]; - } - let parts = argument - .parts - .iter() - .map(|(part, _)| Self::gen_type(&part.ty)) - .collect::>(); - if cfg!(target_arch = "aarch64") && matches!(ty, Type::Struct(_)) && parts.len() > 1 { - vec![format!("[{} x {}]", parts.len(), parts[0])] - } else { - parts - } - } - fn aggregate_pointer( - base: &str, - offset: usize, - options: &mut Options, - lines: &mut Vec, - ) -> String { - if offset == 0 { - return base.into(); - } - let register = options.new_temporary(); - lines.push(format!( - " %{register} = getelementptr i8, ptr {base}, i64 {offset}" - )); - format!("%{register}") - } - fn aggregate_load_part( - part: &crate::generator::abi::Part, - size: usize, - base: &str, - options: &mut Options, - lines: &mut Vec, - ) -> String { - let pointer = Self::aggregate_pointer(base, part.offset, options, lines); - let register = options.new_temporary(); - let remaining = size - part.offset; - let loaded_type = if remaining < part.ty.size_of() { - format!("i{}", remaining * 8) - } else { - Self::gen_type(&part.ty) - }; - lines.push(format!( - " %{register} = load {loaded_type}, ptr {pointer}, align 1" - )); - if remaining < part.ty.size_of() { - let extended = options.new_temporary(); - lines.push(format!( - " %{extended} = zext {loaded_type} %{register} to {}", - Self::gen_type(&part.ty) - )); - format!("%{extended}") - } else { - format!("%{register}") - } - } - fn validate_function_abi(&self, func: &Func) { - let (types, result, _) = func.cty.function_signature_ref().unwrap(); - let aggregate = matches!(result, Type::Struct(_)) - || types.iter().any(|ty| matches!(ty, Type::Struct(_))); - if aggregate - && types - .iter() - .chain(std::iter::once(result)) - .any(Type::contains_extended_long_double) - { - self.error_logger.print_error_position(func.token.as_ref().map_or(0, |token|token.pos), "extended long double cannot be combined with by-value aggregate parameters or results"); - } - } - - fn aggregate_function(&self, name: &str, func: &Func) -> String { - let (types, result, variadic) = func.cty.function_signature_ref().unwrap(); - let signature = Self::aggregate_signature(types, result, types.len(), variadic); - let return_type = if matches!(result, Type::Struct(_)) { - Self::aggregate_result_type(result) - } else { - Self::gen_type(result) - }; - let return_abi = if matches!(result, Type::Struct(_)) { - Cow::Borrowed(return_type.as_str()) - } else { - Cow::Owned(Self::abi_result_type(result)) - }; - let mut parameters = vec![]; - if signature.indirect_result { - parameters.push(format!( - "ptr sret({}) align {} %abi.result", - Self::gen_type(result), - result.align_of() - )); - } - for (index, argument) in signature.arguments.iter().enumerate() { - for (part, ty) in Self::aggregate_parameter_types(&types[index], argument) - .into_iter() - .enumerate() - { - let ty = match Self::abi_extension(&types[index]) { - Some(extension) => format!("{ty} {extension}"), - None => ty, - }; - parameters.push(if func.body.is_some() { - format!("{ty} %abi.arg.{index}.{part}") - } else { - ty - }); - } - } - if variadic { - parameters.push("...".into()); - } - if func.body.is_none() || func.inline_only() { - return format!( - "declare {}{}{return_abi} {}({}){}", - if func.weak_import { "extern_weak " } else { "" }, - if func.hidden && !func.internal { - "hidden " - } else { - "" - }, - self.symbol_name(name), - parameters.join(", "), - if func.noinline { " noinline" } else { "" } - ); - } - let mut number = 0; - let mut registers = HashMap::new(); - let mut queue = vec![]; - let mut options = Options { - register_number: &mut number, - register_map: &mut registers, - register_queue: &mut queue, - label_number: 0, - terminated: false, - return_ty: result.clone(), - return_pointer: signature.indirect_result.then(|| "%abi.result".into()), - break_labels: vec![], - expression_slots: HashMap::new(), - argument_slots: HashMap::new(), - }; - let mut lines = vec![format!( - "define {}{}{return_abi} {}({}){} uwtable \"frame-pointer\"=\"all\" {{", - if func.internal { "internal " } else { "" }, - if func.hidden && !func.internal { - "hidden " - } else { - "" - }, - self.symbol_name(name), - parameters.join(", "), - if func.noinline { " noinline" } else { "" } - )]; - for (index, argument) in signature.arguments.iter().enumerate() { - let ty = &types[index]; - let register = options.new_temporary(); - options - .register_map - .insert(func.args[index].offset.unwrap(), register); - lines.push(format!( - " %{register} = alloca [{} x i8], align {}", - ty.size_of().div_ceil(8) * 8, - ty.align_of() - )); - if argument.memory.is_some() || argument.indirect { - let value = options.new_temporary(); - lines.push(format!( - " %{value} = load {}, ptr %abi.arg.{index}.0, align 1", - Self::gen_type(ty) - )); - lines.push(format!( - " store {} %{value}, ptr %{register}, align {}", - Self::gen_type(ty), - ty.align_of() - )); - } else { - let grouped = cfg!(target_arch = "aarch64") - && matches!(ty, Type::Struct(_)) - && argument.parts.len() > 1; - if argument.parts.is_empty() { - continue; - } - let group_type = Self::aggregate_parameter_types(ty, argument)[0].clone(); - if group_type == "i128" { - lines.push(format!( - " store i128 %abi.arg.{index}.0, ptr %{register}, align 1" - )); - continue; - } - for (part_index, (part, _)) in argument.parts.iter().enumerate() { - let value = if grouped { - let value = options.new_temporary(); - lines.push(format!(" %{value} = extractvalue {group_type} %abi.arg.{index}.0, {part_index}")); - format!("%{value}") - } else { - format!("%abi.arg.{index}.{part_index}") - }; - let pointer = Self::aggregate_pointer( - &format!("%{register}"), - part.offset, - &mut options, - &mut lines, - ); - lines.push(format!( - " store {} {value}, ptr {pointer}, align 1", - Self::gen_type(&part.ty) - )); - } - } - } - Self::allocate_locals(func.body.as_ref().unwrap(), &mut options, &mut lines); - lines.extend(self.gen_node(func.body.as_ref().unwrap(), &mut options)); - if !options.terminated { - lines.push(if return_type == "void" { - " ret void".into() - } else { - format!(" ret {return_type} zeroinitializer") - }); - } - lines.push("}".into()); - lines.join("\n") - } - fn aggregate_call(&self, node: &Node, options: &mut Options) -> Vec { - let types = node - .args - .iter() - .rev() - .map(|arg| arg.resolve_type_view().unwrap()) - .collect::>(); - let result = node.resolve_type_view().unwrap(); - if types.iter().any(|ty| ty.contains_extended_long_double()) - || result.contains_extended_long_double() - { - self.error_logger.print_error_position(node.token.as_ref().map_or(0,|token|token.pos), "extended long double cannot be combined with by-value aggregate arguments or results"); - } - let signature = - Self::aggregate_signature(&types, &result, node.parameter_types.len(), node.variadic); - let mut lines = vec![]; - let callee = if let Some(callee) = &node.lhs { - lines.extend(self.gen_node(callee, options)); - options.register_queue.pop().unwrap() - } else { - self.symbol_name(&node.global_name) - }; - let mut arguments = vec![]; - for (index, arg) in node.args.iter().rev().enumerate() { - let argument = &signature.arguments[index]; - let ty = types[index].as_ref(); - let aggregate = matches!(ty, Type::Struct(_)); - let value = if aggregate { - lines.extend(self.gen_addr(arg, options)); - options.register_queue.pop().unwrap() - } else { - lines.extend(self.gen_node(arg, options)); - options.register_queue.pop().unwrap() - }; - let parameter_types = Self::aggregate_parameter_types(ty, argument); - if parameter_types.is_empty() { - continue; - } - let aligned_variadic = cfg!(all(target_arch = "aarch64", target_os = "macos")) - && node.variadic - && index >= node.parameter_types.len() - && aggregate - && !argument.indirect - && argument.stack_alignment == 16; - if aligned_variadic { - // Unnamed Darwin arguments occupy stack slots. Scalar i128 chunks - // preserve sixteen-byte alignment, including larger HFAs. - for offset in (0..ty.size_of()).step_by(16) { - let pointer = Self::aggregate_pointer(&value, offset, options, &mut lines); - let register = options.new_temporary(); - lines.push(format!(" %{register} = load i128, ptr {pointer}, align 1")); - arguments.push(format!("i128 %{register}")); - } - continue; - } - if argument.memory.is_some() || argument.indirect { - let slot = options.argument_slots[&(node.token.as_ref().unwrap().pos, index)]; - let register = options.new_temporary(); - let alignment = ty.align_of().max(argument.stack_alignment); - lines.push(format!( - " %{register} = load {}, ptr {value}, align 1", - Self::gen_type(ty) - )); - lines.push(format!( - " store {} %{register}, ptr %{slot}, align {alignment}", - Self::gen_type(ty) - )); - arguments.push(format!("{} %{slot}", parameter_types[0])); - } else if aggregate && (parameter_types[0] == "i128") { - let register = options.new_temporary(); - lines.push(format!( - " %{register} = load {}, ptr {value}, align 1", - parameter_types[0] - )); - arguments.push(format!("{} %{register}", parameter_types[0])); - } else { - let mut values = vec![]; - for (part, _) in &argument.parts { - let value = if aggregate { - Self::aggregate_load_part(part, ty.size_of(), &value, options, &mut lines) - } else { - value.clone() - }; - values.push(value); - } - if parameter_types.len() == 1 && values.len() > 1 { - let mut group = "poison".to_string(); - for (index, value) in values.iter().enumerate() { - let register = options.new_temporary(); - lines.push(format!( - " %{register} = insertvalue {} {group}, {} {value}, {index}", - parameter_types[0], - Self::gen_type(&argument.parts[index].0.ty) - )); - group = format!("%{register}"); - } - arguments.push(format!("{} {group}", parameter_types[0])); - } else { - arguments.extend(parameter_types.into_iter().zip(values).map( - |(part_type, value)| { - let parameter_type = match Self::abi_extension(ty) { - Some(extension) => format!("{part_type} {extension}"), - None => part_type, - }; - format!("{parameter_type} {value}") - }, - )); - } - } - } - let result_slot = node - .offset - .map(|offset| format!("%{}", options.register_map[&offset])); - if signature.indirect_result { - arguments.insert( - 0, - format!( - "ptr sret({}) align {} {}", - Self::gen_type(&result), - result.align_of(), - result_slot.as_ref().unwrap() - ), - ); - } - let return_type = if matches!(*result, Type::Struct(_)) { - Self::aggregate_result_type(&result) - } else { - Self::gen_type(&result) - }; - let return_abi = if matches!(*result, Type::Struct(_)) { - Cow::Borrowed(return_type.as_str()) - } else { - Cow::Owned(Self::abi_result_type(&result)) - }; - let variadic_signature = if node.variadic { - let fixed = Self::aggregate_signature( - &node.parameter_types, - &result, - node.parameter_types.len(), - true, - ); - let mut parameters = vec![]; - if fixed.indirect_result { - parameters.push("ptr".into()); - } - for (index, argument) in fixed.arguments.iter().enumerate() { - parameters.extend( - Self::aggregate_parameter_types(&node.parameter_types[index], argument) - .into_iter() - .map(|ty| ty.split(" byval").next().unwrap().to_string()), - ); - } - parameters.push("...".into()); - format!(" ({})", parameters.join(", ")) - } else { - String::new() - }; - let value = if return_type == "void" { - lines.push(format!( - " call void{variadic_signature} {callee}({})", - arguments.join(", ") - )); - "0".into() - } else { - let register = options.new_temporary(); - lines.push(format!( - " %{register} = call {return_abi}{variadic_signature} {callee}({})", - arguments.join(", ") - )); - format!("%{register}") - }; - if matches!(*result, Type::Struct(_)) { - let slot = result_slot.unwrap(); - if !signature.indirect_result { - for (index, part) in signature.result.iter().enumerate() { - let value = if signature.result.len() > 1 { - let register = options.new_temporary(); - lines.push(format!( - " %{register} = extractvalue {return_type} {value}, {index}" - )); - format!("%{register}") - } else { - value.clone() - }; - let pointer = Self::aggregate_pointer(&slot, part.offset, options, &mut lines); - let remaining = result.size_of() - part.offset; - let mut stored_type = Self::gen_type(&part.ty); - let value = if remaining < part.ty.size_of() { - let narrowed = options.new_temporary(); - let narrow_type = format!("i{}", remaining * 8); - lines.push(format!( - " %{narrowed} = trunc {stored_type} {value} to {narrow_type}" - )); - stored_type = narrow_type; - format!("%{narrowed}") - } else { - value - }; - lines.push(format!( - " store {stored_type} {value}, ptr {pointer}, align 1" - )); - } - } - let register = options.new_register(); - lines.push(format!( - " %{register} = load {}, ptr {slot}, align {}", - Self::gen_type(&result), - result.align_of() - )); - } else { - options.register_queue.push(value); - } - lines - } - fn aggregate_return(&self, node: &Node, options: &mut Options) -> Vec { - let ty = options.return_ty.clone(); - let mut lines = self.gen_addr(node.lhs.as_ref().unwrap(), options); - let address = options.register_queue.pop().unwrap(); - if ty.size_of() == 0 { - lines.push(" ret void".into()); - options.terminated = true; - return lines; - } - let parts = crate::generator::abi::aggregate_parts(&ty, &Self::target_arch()); - if let Some(pointer) = options.return_pointer.clone() { - let register = options.new_temporary(); - lines.push(format!( - " %{register} = load {}, ptr {address}, align 1", - Self::gen_type(&ty) - )); - lines.push(format!( - " store {} %{register}, ptr {pointer}, align {}", - Self::gen_type(&ty), - ty.align_of() - )); - lines.push(" ret void".into()); - } else { - let return_type = Self::aggregate_result_type(&ty); - let mut result = "poison".to_string(); - for (index, part) in parts.iter().enumerate() { - let value = - Self::aggregate_load_part(part, ty.size_of(), &address, options, &mut lines); - if parts.len() == 1 { - result = value; - } else { - let combined = options.new_temporary(); - lines.push(format!( - " %{combined} = insertvalue {return_type} {result}, {} {value}, {index}", - Self::gen_type(&part.ty) - )); - result = format!("%{combined}"); - } - } - lines.push(format!(" ret {return_type} {result}")); - } - options.terminated = true; - lines - } fn gen_func(&self, name: &str, func: &Func) -> String { let (types, result, variadic) = func.cty.function_signature_ref().unwrap(); if matches!(result, Type::Struct(_)) || types.iter().any(|ty| matches!(ty, Type::Struct(_))) @@ -1609,172 +736,6 @@ impl<'a> IrGenerator<'a> { } } - fn atomic_update( - node: &Node, - operation: AtomicRmwOperation, - return_new: bool, - values: &[String], - options: &mut Options, - lines: &mut Vec, - ) -> String { - let object = node.args[0].dest_type_view().unwrap(); - let storage = Type::integer((object.size_of() * 8) as u8, false); - let scalar = Self::gen_type(&storage); - let value = Self::convert_integer( - values[1].clone(), - &node.args[1].resolve_type_view().unwrap(), - &storage, - options, - lines, - ); - let mnemonic = match operation { - AtomicRmwOperation::Add => "add", - AtomicRmwOperation::Sub => "sub", - AtomicRmwOperation::And => "and", - AtomicRmwOperation::Or => "or", - AtomicRmwOperation::Xor => "xor", - AtomicRmwOperation::Nand => "nand", - AtomicRmwOperation::Exchange => "xchg", - }; - let volatile = if node.volatile_access { - "volatile " - } else { - "" - }; - let old = options.new_temporary(); - lines.push(format!( - " %{old} = atomicrmw {volatile}{mnemonic} ptr {}, {scalar} {value} seq_cst, align {}", - values[0], - object.align_of() - )); - let result = if return_new { - let result = options.new_temporary(); - if operation == AtomicRmwOperation::Nand { - lines.push(format!(" %{result} = and {scalar} %{old}, {value}")); - let inverted = options.new_temporary(); - lines.push(format!(" %{inverted} = xor {scalar} %{result}, -1")); - format!("%{inverted}") - } else { - lines.push(format!(" %{result} = {mnemonic} {scalar} %{old}, {value}")); - format!("%{result}") - } - } else { - format!("%{old}") - }; - Self::convert_integer(result, &storage, &object, options, lines) - } - - fn atomic_compare_exchange( - node: &Node, - expected_volatile: bool, - values: &[String], - options: &mut Options, - lines: &mut Vec, - ) -> String { - let object = node.args[0].dest_type_view().unwrap(); - let storage = Type::integer((object.size_of() * 8) as u8, false); - let scalar = Self::gen_type(&storage); - let desired = Self::convert_integer( - values[2].clone(), - &node.args[2].resolve_type_view().unwrap(), - &storage, - options, - lines, - ); - let expected = options.new_temporary(); - let expected_volatile = if expected_volatile { "volatile " } else { "" }; - lines.push(format!( - " %{expected} = load {expected_volatile}{scalar}, ptr {}, align {}", - values[1], - object.align_of() - )); - let exchanged = options.new_temporary(); - let volatile = if node.volatile_access { - "volatile " - } else { - "" - }; - lines.push(format!(" %{exchanged} = cmpxchg {volatile}ptr {}, {scalar} %{expected}, {scalar} {desired} seq_cst seq_cst, align {}", values[0], object.align_of())); - let old = options.new_temporary(); - lines.push(format!( - " %{old} = extractvalue {{ {scalar}, i1 }} %{exchanged}, 0" - )); - let success = options.new_temporary(); - lines.push(format!( - " %{success} = extractvalue {{ {scalar}, i1 }} %{exchanged}, 1" - )); - let failure = options.new_label(); - let done = options.new_label(); - lines.push(format!( - " br i1 %{success}, label %{done}, label %{failure}" - )); - lines.push(format!("{failure}:")); - lines.push(format!( - " store {expected_volatile}{scalar} %{old}, ptr {}, align {}", - values[1], - object.align_of() - )); - lines.push(format!(" br label %{done}")); - lines.push(format!("{done}:")); - let result = options.new_temporary(); - lines.push(format!(" %{result} = zext i1 %{success} to i8")); - format!("%{result}") - } - - fn gen_atomic( - &self, - node: &Node, - operation: AtomicOperation, - options: &mut Options, - ) -> Vec { - let mut lines = vec![]; - let mut values = vec![]; - for argument in &node.args { - lines.extend(self.gen_node(argument, options)); - values.push(options.register_queue.pop().unwrap()); - } - if let AtomicOperation::CompareExchange(expected_volatile) = operation { - let result = Self::atomic_compare_exchange( - node, - expected_volatile, - &values, - options, - &mut lines, - ); - options.register_queue.push(result); - return lines; - } - let ty = node.args[0].dest_type_view().unwrap(); - if let AtomicOperation::Update(op, return_new) = operation { - let value = Self::atomic_update(node, op, return_new, &values, options, &mut lines); - options.register_queue.push(value); - return lines; - } - let scalar = Self::gen_type(&ty); - let volatile = if node.volatile_access { - "volatile " - } else { - "" - }; - if operation == AtomicOperation::Load { - lines.push(format!( - " %{} = load atomic {volatile}{scalar}, ptr {} seq_cst, align {}", - options.new_register(), - values[0], - ty.align_of() - )); - } else { - lines.push(format!( - " store atomic {volatile}{scalar} {}, ptr {} seq_cst, align {}", - values[1], - values[0], - ty.align_of() - )); - options.register_queue.push("0".into()); - } - lines - } - fn gen_prefetch( &self, node: &Node, @@ -1792,140 +753,6 @@ impl<'a> IrGenerator<'a> { lines } - fn gen_inline_assembly( - &self, - node: &Node, - volatile: bool, - memory: bool, - options: &mut Options, - ) -> Vec { - let mut lines = vec![]; - let mut matching = vec![]; - for (index, output) in node.children.iter().enumerate() { - if node.value.unwrap() & (1 << index) != 0 { - lines.extend(self.gen_node(output, options)); - let value = options.register_queue.pop().unwrap(); - matching.push(( - index, - format!( - "{} {value}", - Self::gen_type(&output.resolve_type_view().unwrap()) - ), - )); - } - } - let mut arguments = vec![]; - for input in &node.args { - lines.extend(self.gen_node(input, options)); - let value = options.register_queue.pop().unwrap(); - arguments.push(format!( - "{} {value}", - Self::gen_type(&input.resolve_type_view().unwrap()) - )); - } - let template = render_inline_assembly(&node.dest.replace('$', "$$"), |index, modifier| { - let arch = if cfg!(target_arch = "aarch64") { - crate::generator::Arch::Aarch64 - } else { - crate::generator::Arch::X86_64 - }; - let operand = node.children.iter().chain(&node.args).nth(index)?; - crate::generator::asm_register_width( - arch, - modifier, - operand.resolve_type_view()?.size_of(), - )?; - Some(if let Some(modifier) = modifier { - format!("${{{index}:{modifier}}}") - } else { - format!("${index}") - }) - }) - .unwrap_or_else(|| { - self.error_logger.print_error_position( - node.token.as_ref().unwrap().pos, - "unsupported asm register modifier for target", - ); - unreachable!() - }); - let template = template - .bytes() - .map(|byte| format!("\\{byte:02X}")) - .collect::(); - let mut constraints = vec!["=r".to_string(); node.children.len()]; - constraints.extend(vec!["r".to_string(); node.args.len()]); - for (index, value) in matching { - constraints.push(index.to_string()); - arguments.push(value); - } - constraints.extend( - if cfg!(target_arch = "x86_64") { - vec!["~{dirflag}", "~{fpsr}", "~{flags}"] - } else { - vec!["~{cc}"] - } - .into_iter() - .map(String::from), - ); - if memory { - constraints.push("~{memory}".to_string()); - } - let types = node - .children - .iter() - .map(|output| Self::gen_type(&output.resolve_type_view().unwrap())) - .collect::>(); - let result_type = if types.is_empty() { - "void".to_string() - } else if types.len() == 1 { - types[0].clone() - } else { - format!("{{ {} }}", types.join(", ")) - }; - let result = if types.is_empty() { - String::new() - } else { - let register = format!("%{}", options.new_temporary()); - register - }; - let assignment = if types.is_empty() { - String::new() - } else { - format!("{result} = ") - }; - lines.push(format!( - " {assignment}call {result_type} asm {}\"{template}\", \"{}\"({})", - if volatile { "sideeffect " } else { "" }, - constraints.join(","), - arguments.join(", ") - )); - for (index, output) in node.children.iter().enumerate() { - let value = if types.len() == 1 { - result.clone() - } else { - let value = format!("%{}", options.new_temporary()); - lines.push(format!( - " {value} = extractvalue {result_type} {result}, {index}" - )); - value - }; - lines.extend(self.gen_addr(output, options)); - let address = options.register_queue.pop().unwrap(); - lines.push(format!( - " store {}{} {value}, ptr {address}, align {}", - if output.volatile_access { - "volatile " - } else { - "" - }, - types[index], - Self::access_alignment(output) - )); - } - options.register_queue.push("0".to_string()); - lines - } - fn gen_node(&self, node: &Node, options: &mut Options) -> Vec { if options.terminated && !matches!(node.nt, NodeType::Block | NodeType::Case | NodeType::Label) @@ -2840,6 +1667,7 @@ impl<'a> IrGenerator<'a> { #[cfg(test)] mod tests { use super::*; + use crate::global::GlobalVariableData; use std::rc::Rc; #[test] diff --git a/src/generator/llvm/generator/aggregate.rs b/src/generator/llvm/generator/aggregate.rs new file mode 100644 index 0000000..b802fab --- /dev/null +++ b/src/generator/llvm/generator/aggregate.rs @@ -0,0 +1,575 @@ +use super::IrGenerator; +use super::Options; +use crate::ctype::Type; +use crate::func::Func; +use crate::node::Node; +use std::borrow::Cow; +use std::collections::HashMap; + +impl IrGenerator<'_> { + pub(super) fn aggregate_result_type(ty: &Type) -> String { + let parts = crate::generator::abi::aggregate_parts(ty, &Self::target_arch()); + if parts.is_empty() { + "void".into() + } else if parts.len() == 1 { + Self::gen_type(&parts[0].ty) + } else { + format!( + "{{ {} }}", + parts + .iter() + .map(|part| Self::gen_type(&part.ty)) + .collect::>() + .join(", ") + ) + } + } + + pub(super) fn aggregate_signature( + types: &[impl std::borrow::Borrow], + result: &Type, + fixed: usize, + variadic: bool, + ) -> crate::generator::abi::Signature { + use crate::generator::abi; + let mut signature = abi::signature( + types, + result, + Self::target_arch(), + Self::target_os(), + fixed, + variadic, + ); + if cfg!(all(target_arch = "aarch64", target_os = "macos")) { + for (ty, argument) in types.iter().zip(&mut signature.arguments) { + if let Some(offset) = argument.memory { + let parts = + abi::aggregate_parts(std::borrow::Borrow::borrow(ty), &Self::target_arch()); + if !parts.is_empty() { + // AArch64 lowers direct aggregates using the same coercion + // whether they occupy registers or stack slots. LLVM byval + // has different stack-slot accounting on Darwin. + argument.parts = parts + .into_iter() + .map(|part| (part, abi::Location::Stack(offset))) + .collect(); + argument.memory = None; + } + } + } + } + signature + } + + pub(super) fn aggregate_parameter_types( + ty: &Type, + argument: &crate::generator::abi::Argument, + ) -> Vec { + if argument.memory.is_some() { + return vec![format!( + "ptr byval({}) align {}", + Self::gen_type(ty), + argument.stack_alignment + )]; + } + if argument.indirect { + return vec!["ptr".into()]; + } + if cfg!(target_arch = "aarch64") + && matches!(ty.unqualified(), Type::Struct(_)) + && (if cfg!(target_os = "linux") { + ty.natural_align_of() + } else { + ty.align_of() + }) >= 16 + && argument.parts.len() == 2 + && argument + .parts + .iter() + .all(|(part, _)| part.ty.float_bits().is_none()) + { + return vec!["i128".into()]; + } + let parts = argument + .parts + .iter() + .map(|(part, _)| Self::gen_type(&part.ty)) + .collect::>(); + if cfg!(target_arch = "aarch64") && matches!(ty, Type::Struct(_)) && parts.len() > 1 { + vec![format!("[{} x {}]", parts.len(), parts[0])] + } else { + parts + } + } + + pub(super) fn aggregate_pointer( + base: &str, + offset: usize, + options: &mut Options, + lines: &mut Vec, + ) -> String { + if offset == 0 { + return base.into(); + } + let register = options.new_temporary(); + lines.push(format!( + " %{register} = getelementptr i8, ptr {base}, i64 {offset}" + )); + format!("%{register}") + } + + pub(super) fn aggregate_load_part( + part: &crate::generator::abi::Part, + size: usize, + base: &str, + options: &mut Options, + lines: &mut Vec, + ) -> String { + let pointer = Self::aggregate_pointer(base, part.offset, options, lines); + let register = options.new_temporary(); + let remaining = size - part.offset; + let loaded_type = if remaining < part.ty.size_of() { + format!("i{}", remaining * 8) + } else { + Self::gen_type(&part.ty) + }; + lines.push(format!( + " %{register} = load {loaded_type}, ptr {pointer}, align 1" + )); + if remaining < part.ty.size_of() { + let extended = options.new_temporary(); + lines.push(format!( + " %{extended} = zext {loaded_type} %{register} to {}", + Self::gen_type(&part.ty) + )); + format!("%{extended}") + } else { + format!("%{register}") + } + } + + pub(super) fn validate_function_abi(&self, func: &Func) { + let (types, result, _) = func.cty.function_signature_ref().unwrap(); + let aggregate = matches!(result, Type::Struct(_)) + || types.iter().any(|ty| matches!(ty, Type::Struct(_))); + if aggregate + && types + .iter() + .chain(std::iter::once(result)) + .any(Type::contains_extended_long_double) + { + self.error_logger.print_error_position(func.token.as_ref().map_or(0, |token|token.pos), "extended long double cannot be combined with by-value aggregate parameters or results"); + } + } + + pub(super) fn aggregate_function(&self, name: &str, func: &Func) -> String { + let (types, result, variadic) = func.cty.function_signature_ref().unwrap(); + let signature = Self::aggregate_signature(types, result, types.len(), variadic); + let return_type = if matches!(result, Type::Struct(_)) { + Self::aggregate_result_type(result) + } else { + Self::gen_type(result) + }; + let return_abi = if matches!(result, Type::Struct(_)) { + Cow::Borrowed(return_type.as_str()) + } else { + Cow::Owned(Self::abi_result_type(result)) + }; + let mut parameters = vec![]; + if signature.indirect_result { + parameters.push(format!( + "ptr sret({}) align {} %abi.result", + Self::gen_type(result), + result.align_of() + )); + } + for (index, argument) in signature.arguments.iter().enumerate() { + for (part, ty) in Self::aggregate_parameter_types(&types[index], argument) + .into_iter() + .enumerate() + { + let ty = match Self::abi_extension(&types[index]) { + Some(extension) => format!("{ty} {extension}"), + None => ty, + }; + parameters.push(if func.body.is_some() { + format!("{ty} %abi.arg.{index}.{part}") + } else { + ty + }); + } + } + if variadic { + parameters.push("...".into()); + } + if func.body.is_none() || func.inline_only() { + return format!( + "declare {}{}{return_abi} {}({}){}", + if func.weak_import { "extern_weak " } else { "" }, + if func.hidden && !func.internal { + "hidden " + } else { + "" + }, + self.symbol_name(name), + parameters.join(", "), + if func.noinline { " noinline" } else { "" } + ); + } + let mut number = 0; + let mut registers = HashMap::new(); + let mut queue = vec![]; + let mut options = Options { + register_number: &mut number, + register_map: &mut registers, + register_queue: &mut queue, + label_number: 0, + terminated: false, + return_ty: result.clone(), + return_pointer: signature.indirect_result.then(|| "%abi.result".into()), + break_labels: vec![], + expression_slots: HashMap::new(), + argument_slots: HashMap::new(), + }; + let mut lines = vec![format!( + "define {}{}{return_abi} {}({}){} uwtable \"frame-pointer\"=\"all\" {{", + if func.internal { "internal " } else { "" }, + if func.hidden && !func.internal { + "hidden " + } else { + "" + }, + self.symbol_name(name), + parameters.join(", "), + if func.noinline { " noinline" } else { "" } + )]; + for (index, argument) in signature.arguments.iter().enumerate() { + let ty = &types[index]; + let register = options.new_temporary(); + options + .register_map + .insert(func.args[index].offset.unwrap(), register); + lines.push(format!( + " %{register} = alloca [{} x i8], align {}", + ty.size_of().div_ceil(8) * 8, + ty.align_of() + )); + if argument.memory.is_some() || argument.indirect { + let value = options.new_temporary(); + lines.push(format!( + " %{value} = load {}, ptr %abi.arg.{index}.0, align 1", + Self::gen_type(ty) + )); + lines.push(format!( + " store {} %{value}, ptr %{register}, align {}", + Self::gen_type(ty), + ty.align_of() + )); + } else { + let grouped = cfg!(target_arch = "aarch64") + && matches!(ty, Type::Struct(_)) + && argument.parts.len() > 1; + if argument.parts.is_empty() { + continue; + } + let group_type = Self::aggregate_parameter_types(ty, argument)[0].clone(); + if group_type == "i128" { + lines.push(format!( + " store i128 %abi.arg.{index}.0, ptr %{register}, align 1" + )); + continue; + } + for (part_index, (part, _)) in argument.parts.iter().enumerate() { + let value = if grouped { + let value = options.new_temporary(); + lines.push(format!(" %{value} = extractvalue {group_type} %abi.arg.{index}.0, {part_index}")); + format!("%{value}") + } else { + format!("%abi.arg.{index}.{part_index}") + }; + let pointer = Self::aggregate_pointer( + &format!("%{register}"), + part.offset, + &mut options, + &mut lines, + ); + lines.push(format!( + " store {} {value}, ptr {pointer}, align 1", + Self::gen_type(&part.ty) + )); + } + } + } + Self::allocate_locals(func.body.as_ref().unwrap(), &mut options, &mut lines); + lines.extend(self.gen_node(func.body.as_ref().unwrap(), &mut options)); + if !options.terminated { + lines.push(if return_type == "void" { + " ret void".into() + } else { + format!(" ret {return_type} zeroinitializer") + }); + } + lines.push("}".into()); + lines.join("\n") + } + + pub(super) fn aggregate_call(&self, node: &Node, options: &mut Options) -> Vec { + let types = node + .args + .iter() + .rev() + .map(|arg| arg.resolve_type_view().unwrap()) + .collect::>(); + let result = node.resolve_type_view().unwrap(); + if types.iter().any(|ty| ty.contains_extended_long_double()) + || result.contains_extended_long_double() + { + self.error_logger.print_error_position(node.token.as_ref().map_or(0,|token|token.pos), "extended long double cannot be combined with by-value aggregate arguments or results"); + } + let signature = + Self::aggregate_signature(&types, &result, node.parameter_types.len(), node.variadic); + let mut lines = vec![]; + let callee = if let Some(callee) = &node.lhs { + lines.extend(self.gen_node(callee, options)); + options.register_queue.pop().unwrap() + } else { + self.symbol_name(&node.global_name) + }; + let mut arguments = vec![]; + for (index, arg) in node.args.iter().rev().enumerate() { + let argument = &signature.arguments[index]; + let ty = types[index].as_ref(); + let aggregate = matches!(ty, Type::Struct(_)); + let value = if aggregate { + lines.extend(self.gen_addr(arg, options)); + options.register_queue.pop().unwrap() + } else { + lines.extend(self.gen_node(arg, options)); + options.register_queue.pop().unwrap() + }; + let parameter_types = Self::aggregate_parameter_types(ty, argument); + if parameter_types.is_empty() { + continue; + } + let aligned_variadic = cfg!(all(target_arch = "aarch64", target_os = "macos")) + && node.variadic + && index >= node.parameter_types.len() + && aggregate + && !argument.indirect + && argument.stack_alignment == 16; + if aligned_variadic { + // Unnamed Darwin arguments occupy stack slots. Scalar i128 chunks + // preserve sixteen-byte alignment, including larger HFAs. + for offset in (0..ty.size_of()).step_by(16) { + let pointer = Self::aggregate_pointer(&value, offset, options, &mut lines); + let register = options.new_temporary(); + lines.push(format!(" %{register} = load i128, ptr {pointer}, align 1")); + arguments.push(format!("i128 %{register}")); + } + continue; + } + if argument.memory.is_some() || argument.indirect { + let slot = options.argument_slots[&(node.token.as_ref().unwrap().pos, index)]; + let register = options.new_temporary(); + let alignment = ty.align_of().max(argument.stack_alignment); + lines.push(format!( + " %{register} = load {}, ptr {value}, align 1", + Self::gen_type(ty) + )); + lines.push(format!( + " store {} %{register}, ptr %{slot}, align {alignment}", + Self::gen_type(ty) + )); + arguments.push(format!("{} %{slot}", parameter_types[0])); + } else if aggregate && (parameter_types[0] == "i128") { + let register = options.new_temporary(); + lines.push(format!( + " %{register} = load {}, ptr {value}, align 1", + parameter_types[0] + )); + arguments.push(format!("{} %{register}", parameter_types[0])); + } else { + let mut values = vec![]; + for (part, _) in &argument.parts { + let value = if aggregate { + Self::aggregate_load_part(part, ty.size_of(), &value, options, &mut lines) + } else { + value.clone() + }; + values.push(value); + } + if parameter_types.len() == 1 && values.len() > 1 { + let mut group = "poison".to_string(); + for (index, value) in values.iter().enumerate() { + let register = options.new_temporary(); + lines.push(format!( + " %{register} = insertvalue {} {group}, {} {value}, {index}", + parameter_types[0], + Self::gen_type(&argument.parts[index].0.ty) + )); + group = format!("%{register}"); + } + arguments.push(format!("{} {group}", parameter_types[0])); + } else { + arguments.extend(parameter_types.into_iter().zip(values).map( + |(part_type, value)| { + let parameter_type = match Self::abi_extension(ty) { + Some(extension) => format!("{part_type} {extension}"), + None => part_type, + }; + format!("{parameter_type} {value}") + }, + )); + } + } + } + let result_slot = node + .offset + .map(|offset| format!("%{}", options.register_map[&offset])); + if signature.indirect_result { + arguments.insert( + 0, + format!( + "ptr sret({}) align {} {}", + Self::gen_type(&result), + result.align_of(), + result_slot.as_ref().unwrap() + ), + ); + } + let return_type = if matches!(*result, Type::Struct(_)) { + Self::aggregate_result_type(&result) + } else { + Self::gen_type(&result) + }; + let return_abi = if matches!(*result, Type::Struct(_)) { + Cow::Borrowed(return_type.as_str()) + } else { + Cow::Owned(Self::abi_result_type(&result)) + }; + let variadic_signature = if node.variadic { + let fixed = Self::aggregate_signature( + &node.parameter_types, + &result, + node.parameter_types.len(), + true, + ); + let mut parameters = vec![]; + if fixed.indirect_result { + parameters.push("ptr".into()); + } + for (index, argument) in fixed.arguments.iter().enumerate() { + parameters.extend( + Self::aggregate_parameter_types(&node.parameter_types[index], argument) + .into_iter() + .map(|ty| ty.split(" byval").next().unwrap().to_string()), + ); + } + parameters.push("...".into()); + format!(" ({})", parameters.join(", ")) + } else { + String::new() + }; + let value = if return_type == "void" { + lines.push(format!( + " call void{variadic_signature} {callee}({})", + arguments.join(", ") + )); + "0".into() + } else { + let register = options.new_temporary(); + lines.push(format!( + " %{register} = call {return_abi}{variadic_signature} {callee}({})", + arguments.join(", ") + )); + format!("%{register}") + }; + if matches!(*result, Type::Struct(_)) { + let slot = result_slot.unwrap(); + if !signature.indirect_result { + for (index, part) in signature.result.iter().enumerate() { + let value = if signature.result.len() > 1 { + let register = options.new_temporary(); + lines.push(format!( + " %{register} = extractvalue {return_type} {value}, {index}" + )); + format!("%{register}") + } else { + value.clone() + }; + let pointer = Self::aggregate_pointer(&slot, part.offset, options, &mut lines); + let remaining = result.size_of() - part.offset; + let mut stored_type = Self::gen_type(&part.ty); + let value = if remaining < part.ty.size_of() { + let narrowed = options.new_temporary(); + let narrow_type = format!("i{}", remaining * 8); + lines.push(format!( + " %{narrowed} = trunc {stored_type} {value} to {narrow_type}" + )); + stored_type = narrow_type; + format!("%{narrowed}") + } else { + value + }; + lines.push(format!( + " store {stored_type} {value}, ptr {pointer}, align 1" + )); + } + } + let register = options.new_register(); + lines.push(format!( + " %{register} = load {}, ptr {slot}, align {}", + Self::gen_type(&result), + result.align_of() + )); + } else { + options.register_queue.push(value); + } + lines + } + + pub(super) fn aggregate_return(&self, node: &Node, options: &mut Options) -> Vec { + let ty = options.return_ty.clone(); + let mut lines = self.gen_addr(node.lhs.as_ref().unwrap(), options); + let address = options.register_queue.pop().unwrap(); + if ty.size_of() == 0 { + lines.push(" ret void".into()); + options.terminated = true; + return lines; + } + let parts = crate::generator::abi::aggregate_parts(&ty, &Self::target_arch()); + if let Some(pointer) = options.return_pointer.clone() { + let register = options.new_temporary(); + lines.push(format!( + " %{register} = load {}, ptr {address}, align 1", + Self::gen_type(&ty) + )); + lines.push(format!( + " store {} %{register}, ptr {pointer}, align {}", + Self::gen_type(&ty), + ty.align_of() + )); + lines.push(" ret void".into()); + } else { + let return_type = Self::aggregate_result_type(&ty); + let mut result = "poison".to_string(); + for (index, part) in parts.iter().enumerate() { + let value = + Self::aggregate_load_part(part, ty.size_of(), &address, options, &mut lines); + if parts.len() == 1 { + result = value; + } else { + let combined = options.new_temporary(); + lines.push(format!( + " %{combined} = insertvalue {return_type} {result}, {} {value}, {index}", + Self::gen_type(&part.ty) + )); + result = format!("%{combined}"); + } + } + lines.push(format!(" ret {return_type} {result}")); + } + options.terminated = true; + lines + } +} diff --git a/src/generator/llvm/generator/atomic.rs b/src/generator/llvm/generator/atomic.rs new file mode 100644 index 0000000..61a2a1b --- /dev/null +++ b/src/generator/llvm/generator/atomic.rs @@ -0,0 +1,172 @@ +use super::IrGenerator; +use super::Options; +use crate::ctype::Type; +use crate::node::{AtomicOperation, AtomicRmwOperation, Node}; + +impl IrGenerator<'_> { + pub(super) fn atomic_update( + node: &Node, + operation: AtomicRmwOperation, + return_new: bool, + values: &[String], + options: &mut Options, + lines: &mut Vec, + ) -> String { + let object = node.args[0].dest_type_view().unwrap(); + let storage = Type::integer((object.size_of() * 8) as u8, false); + let scalar = Self::gen_type(&storage); + let value = Self::convert_integer( + values[1].clone(), + &node.args[1].resolve_type_view().unwrap(), + &storage, + options, + lines, + ); + let mnemonic = match operation { + AtomicRmwOperation::Add => "add", + AtomicRmwOperation::Sub => "sub", + AtomicRmwOperation::And => "and", + AtomicRmwOperation::Or => "or", + AtomicRmwOperation::Xor => "xor", + AtomicRmwOperation::Nand => "nand", + AtomicRmwOperation::Exchange => "xchg", + }; + let volatile = if node.volatile_access { + "volatile " + } else { + "" + }; + let old = options.new_temporary(); + lines.push(format!( + " %{old} = atomicrmw {volatile}{mnemonic} ptr {}, {scalar} {value} seq_cst, align {}", + values[0], + object.align_of() + )); + let result = if return_new { + let result = options.new_temporary(); + if operation == AtomicRmwOperation::Nand { + lines.push(format!(" %{result} = and {scalar} %{old}, {value}")); + let inverted = options.new_temporary(); + lines.push(format!(" %{inverted} = xor {scalar} %{result}, -1")); + format!("%{inverted}") + } else { + lines.push(format!(" %{result} = {mnemonic} {scalar} %{old}, {value}")); + format!("%{result}") + } + } else { + format!("%{old}") + }; + Self::convert_integer(result, &storage, &object, options, lines) + } + + pub(super) fn atomic_compare_exchange( + node: &Node, + expected_volatile: bool, + values: &[String], + options: &mut Options, + lines: &mut Vec, + ) -> String { + let object = node.args[0].dest_type_view().unwrap(); + let storage = Type::integer((object.size_of() * 8) as u8, false); + let scalar = Self::gen_type(&storage); + let desired = Self::convert_integer( + values[2].clone(), + &node.args[2].resolve_type_view().unwrap(), + &storage, + options, + lines, + ); + let expected = options.new_temporary(); + let expected_volatile = if expected_volatile { "volatile " } else { "" }; + lines.push(format!( + " %{expected} = load {expected_volatile}{scalar}, ptr {}, align {}", + values[1], + object.align_of() + )); + let exchanged = options.new_temporary(); + let volatile = if node.volatile_access { + "volatile " + } else { + "" + }; + lines.push(format!(" %{exchanged} = cmpxchg {volatile}ptr {}, {scalar} %{expected}, {scalar} {desired} seq_cst seq_cst, align {}", values[0], object.align_of())); + let old = options.new_temporary(); + lines.push(format!( + " %{old} = extractvalue {{ {scalar}, i1 }} %{exchanged}, 0" + )); + let success = options.new_temporary(); + lines.push(format!( + " %{success} = extractvalue {{ {scalar}, i1 }} %{exchanged}, 1" + )); + let failure = options.new_label(); + let done = options.new_label(); + lines.push(format!( + " br i1 %{success}, label %{done}, label %{failure}" + )); + lines.push(format!("{failure}:")); + lines.push(format!( + " store {expected_volatile}{scalar} %{old}, ptr {}, align {}", + values[1], + object.align_of() + )); + lines.push(format!(" br label %{done}")); + lines.push(format!("{done}:")); + let result = options.new_temporary(); + lines.push(format!(" %{result} = zext i1 %{success} to i8")); + format!("%{result}") + } + + pub(super) fn gen_atomic( + &self, + node: &Node, + operation: AtomicOperation, + options: &mut Options, + ) -> Vec { + let mut lines = vec![]; + let mut values = vec![]; + for argument in &node.args { + lines.extend(self.gen_node(argument, options)); + values.push(options.register_queue.pop().unwrap()); + } + if let AtomicOperation::CompareExchange(expected_volatile) = operation { + let result = Self::atomic_compare_exchange( + node, + expected_volatile, + &values, + options, + &mut lines, + ); + options.register_queue.push(result); + return lines; + } + let ty = node.args[0].dest_type_view().unwrap(); + if let AtomicOperation::Update(op, return_new) = operation { + let value = Self::atomic_update(node, op, return_new, &values, options, &mut lines); + options.register_queue.push(value); + return lines; + } + let scalar = Self::gen_type(&ty); + let volatile = if node.volatile_access { + "volatile " + } else { + "" + }; + if operation == AtomicOperation::Load { + lines.push(format!( + " %{} = load atomic {volatile}{scalar}, ptr {} seq_cst, align {}", + options.new_register(), + values[0], + ty.align_of() + )); + } else { + lines.push(format!( + " store atomic {volatile}{scalar} {}, ptr {} seq_cst, align {}", + values[1], + values[0], + ty.align_of() + )); + options.register_queue.push("0".into()); + } + lines + } +} diff --git a/src/generator/llvm/generator/globals.rs b/src/generator/llvm/generator/globals.rs new file mode 100644 index 0000000..676f8bf --- /dev/null +++ b/src/generator/llvm/generator/globals.rs @@ -0,0 +1,333 @@ +use super::IrGenerator; +use crate::ctype::Type; +use crate::global::{GlobalVariable, GlobalVariableData}; +use crate::token::Token; +use std::collections::HashMap; + +impl IrGenerator<'_> { + pub(super) fn gen_string_literals(&self, string_literals: &[Token]) -> Vec { + string_literals + .iter() + .enumerate() + .map(|(i, token)| { + let bytes = token.encoded_string(); + let width = token.string_encoding.unit_size(); + let element_type = token.string_encoding.element_type(); + let initializer = if width == 1 { + format!( + "c\"{}\"", + bytes.iter().map(|byte| format!("\\{:02X}", byte)).collect::() + ) + } else { + let elements = bytes.chunks_exact(width).map(|chunk| { + let mut value = [0u8; 4]; + value[..width].copy_from_slice(chunk); + format!("{element_type} {}", u32::from_le_bytes(value)) + }).collect::>().join(", "); + format!("[{elements}]") + }; + format!( + "@.str.{i} = private unnamed_addr constant [{} x {element_type}] {initializer}, align {width}", + bytes.len() / width, + ) + }) + .collect::>() + } + + pub(super) fn gen_global_variable(&self, name: &str, gv: &GlobalVariable) -> String { + if gv.declaration_only { + // Opaque pointers can reference an external record before its layout is known. + let incomplete_record = matches!(gv.ty.unqualified(), Type::Struct(record) if !record.definition().borrow().complete); + let ty = if incomplete_record { + "i8".into() + } else { + Self::gen_type(&gv.ty) + }; + return format!( + "{} = {} {}{}global {}, align {}", + self.symbol_name(name), + if gv.weak_import { + "extern_weak" + } else { + "external" + }, + if gv.hidden && !gv.internal { + "hidden " + } else { + "" + }, + if gv.thread_local { "thread_local " } else { "" }, + ty, + Self::align(&gv.ty).max(gv.alignment).max(1) + ); + } + format!( + "{} = {}{}{}global {}, align {}", + self.symbol_name(name), + if gv.internal { + "internal " + } else if gv.common && !gv.thread_local && gv.data.is_none() { + "common " + } else { + "" + }, + if gv.hidden && !gv.internal { + "hidden " + } else { + "" + }, + if gv.thread_local { "thread_local " } else { "" }, + if Self::contains_packed_record(&gv.ty) { + self.packed_global_initializer(&gv.ty, gv.data.as_ref()) + } else { + self.gen_initializer_element(&gv.ty, gv.data.as_ref()) + }, + Self::align(&gv.ty).max(gv.alignment), + ) + } + + pub(super) fn contains_packed_record(ty: &Type) -> bool { + match ty.unqualified() { + Type::Struct(record) => { + record.definition().borrow().packing.is_some() + || ty.contains_typedef_alignment() + || record + .members() + .iter() + .any(|member| Self::contains_packed_record(&member.ty)) + } + Type::Arr(child, _) => Self::contains_packed_record(child), + _ => false, + } + } + + pub(super) fn packed_global_initializer( + &self, + ty: &Type, + data: Option<&GlobalVariableData>, + ) -> String { + let mut bytes = vec![0; ty.size_of()]; + let mut addresses = HashMap::new(); + crate::global::initializer_bytes(ty, data, 0, &mut bytes, &mut addresses); + let mut addresses = addresses.into_iter().collect::>(); + addresses.sort_by_key(|(offset, _)| *offset); + let mut types = vec![]; + let mut values = vec![]; + let mut cursor = 0; + for (offset, (symbol, delta)) in addresses { + if offset > cursor { + let count = offset - cursor; + types.push(format!("[{count} x i8]")); + values.push(format!( + "[{count} x i8] c\"{}\"", + bytes[cursor..offset] + .iter() + .map(|byte| format!("\\{byte:02X}")) + .collect::() + )); + } + types.push("ptr".into()); + let pointer = if delta == 0 { + self.symbol_name(&symbol) + } else { + format!( + "getelementptr (i8, ptr {}, i64 {delta})", + self.symbol_name(&symbol) + ) + }; + values.push(format!("ptr {pointer}")); + cursor = offset + 8; + } + if cursor < bytes.len() { + let count = bytes.len() - cursor; + types.push(format!("[{count} x i8]")); + values.push(format!( + "[{count} x i8] c\"{}\"", + bytes[cursor..] + .iter() + .map(|byte| format!("\\{byte:02X}")) + .collect::() + )); + } + // Opaque pointers allow the definition to use an exact packed storage + // type, including relocations at offsets below pointer alignment. + format!("<{{ {} }}> <{{ {} }}>", types.join(", "), values.join(", ")) + } + + pub(super) fn gen_initializer_element( + &self, + ty: &Type, + data: Option<&GlobalVariableData>, + ) -> String { + let data = if !matches!(ty, Type::Arr(..) | Type::Struct(_)) { + let mut value = data; + while let Some(GlobalVariableData::Arr(elements)) = value { + value = elements.first(); + } + value + } else { + data + }; + if ty.integer_info().is_some_and(|(bits, _)| bits == 64) { + if let Some(GlobalVariableData::Address { symbol, offset }) = data { + let pointer = if *offset == 0 { + self.symbol_name(symbol) + } else { + format!( + "getelementptr (i8, ptr {}, i64 {offset})", + self.symbol_name(symbol) + ) + }; + return format!("i64 ptrtoint (ptr {pointer} to i64)"); + } + } + match ty { + Type::Aligned(child, _) => self.gen_initializer_element(child, data), + Type::Vla(..) => unreachable!("variable length arrays have automatic storage"), + Type::Integer128 { .. } => "<2 x i64> zeroinitializer".into(), + Type::Float16 => "half zeroinitializer".into(), + Type::Vector(..) => format!("{} zeroinitializer", Self::gen_type(ty)), + Type::LongDouble(80) => { + let bits = if let Some(GlobalVariableData::Elem(value)) = data { + value.parse::().unwrap() + } else { + 0 + }; + format!("x86_fp80 {}", crate::floating::llvm_literal(bits)) + } + Type::LongDouble(128) => { + format!("{} zeroinitializer", Self::gen_type(ty)) + } + Type::Float(_) | Type::LongDouble(_) => { + let bits = if let Some(GlobalVariableData::Elem(value)) = data { + value.parse::().unwrap() as usize + } else { + 0 + }; + format!("{} {}", Self::gen_type(ty), Self::float_literal(ty, bits)) + } + + Type::Struct(struct_type) + if struct_type.definition().borrow().is_union || struct_type.has_bit_fields() => + { + let mut bytes = vec![0u8; ty.size_of()]; + let mut addresses = HashMap::new(); + crate::global::initializer_bytes(ty, data, 0, &mut bytes, &mut addresses); + let width = ty.align_of().min(8); + let elements = bytes + .chunks(width) + .enumerate() + .map(|(index, chunk)| { + let mut word = [0u8; 8]; + word[..chunk.len()].copy_from_slice(chunk); + if let Some((symbol, offset)) = addresses.get(&(index * width)) { + let pointer = if *offset == 0 { + self.symbol_name(symbol) + } else { + format!( + "getelementptr (i8, ptr {}, i64 {offset})", + self.symbol_name(symbol) + ) + }; + format!("i64 ptrtoint (ptr {pointer} to i64)") + } else { + format!("i{} {}", width * 8, u64::from_le_bytes(word)) + } + }) + .collect::>() + .join(", "); + if ty.align_of() > width { + format!( + "{} {{ [{} x i{}] [{}], [0 x {}] zeroinitializer }}", + Self::gen_type(ty), + ty.size_of() / width, + width * 8, + elements, + Self::alignment_type(ty.align_of()), + ) + } else { + format!("{} [{}]", Self::gen_type(ty), elements) + } + } + Type::Qualified(child, _) => self.gen_initializer_element(child, data), + Type::Struct(struct_type) => { + if let Some(GlobalVariableData::Arr(elements)) = data { + let mut fields = vec![]; + let mut offset = 0usize; + for (index, member) in struct_type.members().iter().enumerate() { + let aligned_offset = + offset.div_ceil(member.ty.align_of()) * member.ty.align_of(); + if member.offset > aligned_offset { + let padding = format!("[{} x i8]", member.offset - aligned_offset); + fields.push(format!("{padding} zeroinitializer")); + } + fields.push(self.gen_initializer_element(&member.ty, elements.get(index))); + offset = member.offset + member.ty.size_of(); + } + if ty.size_of() > offset { + let padding = format!("[{} x i8]", ty.size_of() - offset); + fields.push(format!("{padding} zeroinitializer")); + } + if let Some(anchor) = Self::record_alignment_member(struct_type) { + fields.push(format!("{anchor} zeroinitializer")); + } + format!("{} {{ {} }}", Self::gen_type(ty), fields.join(", ")) + } else { + format!("{} zeroinitializer", Self::gen_type(ty)) + } + } + Type::Arr(children_ty, size) => { + if let Some(GlobalVariableData::Arr(v)) = data { + format!( + "{} [{}]", + Self::gen_type(ty), + (0..size.unwrap_or(0)) + .map(|i| self.gen_initializer_element(children_ty, v.get(i))) + .collect::>() + .join(", "), + ) + } else { + format!("{} zeroinitializer", Self::gen_type(ty)) + } + } + Type::Ptr(_) => format!( + "ptr {}", + match data { + Some(GlobalVariableData::Address { symbol, offset }) => { + if *offset == 0 { + self.symbol_name(symbol) + } else { + format!( + "getelementptr (i8, ptr {}, i64 {offset})", + self.symbol_name(symbol) + ) + } + } + Some(GlobalVariableData::Elem(value)) if value.starts_with('@') => + value.clone(), + Some(GlobalVariableData::Elem(value)) if value != "0" => + format!("inttoptr (i64 {} to ptr)", value), + _ => "null".to_string(), + } + ), + Type::Enum(value) => self.gen_initializer_element(&value.underlying.borrow(), data), + Type::I8 + | Type::SignedChar + | Type::I32 + | Type::LongLong { .. } + | Type::Integer { .. } + | Type::Bool => format!( + "{} {}", + Self::gen_type(ty), + if let Some(GlobalVariableData::Elem(s)) = data { + s + } else { + "0" + } + ), + Type::Void | Type::Func(_, _) | Type::VariadicFunc(_, _) => { + format!("{} zeroinitializer", Self::gen_type(ty)) + } + } + } +} diff --git a/src/generator/llvm/generator/inline_asm.rs b/src/generator/llvm/generator/inline_asm.rs new file mode 100644 index 0000000..586a8f5 --- /dev/null +++ b/src/generator/llvm/generator/inline_asm.rs @@ -0,0 +1,139 @@ +use super::IrGenerator; +use super::Options; +use crate::node::{Node, render_inline_assembly}; + +impl IrGenerator<'_> { + pub(super) fn gen_inline_assembly( + &self, + node: &Node, + volatile: bool, + memory: bool, + options: &mut Options, + ) -> Vec { + let mut lines = vec![]; + let mut matching = vec![]; + for (index, output) in node.children.iter().enumerate() { + if node.value.unwrap() & (1 << index) != 0 { + lines.extend(self.gen_node(output, options)); + let value = options.register_queue.pop().unwrap(); + matching.push(( + index, + format!( + "{} {value}", + Self::gen_type(&output.resolve_type_view().unwrap()) + ), + )); + } + } + let mut arguments = vec![]; + for input in &node.args { + lines.extend(self.gen_node(input, options)); + let value = options.register_queue.pop().unwrap(); + arguments.push(format!( + "{} {value}", + Self::gen_type(&input.resolve_type_view().unwrap()) + )); + } + let template = render_inline_assembly(&node.dest.replace('$', "$$"), |index, modifier| { + let arch = if cfg!(target_arch = "aarch64") { + crate::generator::Arch::Aarch64 + } else { + crate::generator::Arch::X86_64 + }; + let operand = node.children.iter().chain(&node.args).nth(index)?; + crate::generator::asm_register_width( + arch, + modifier, + operand.resolve_type_view()?.size_of(), + )?; + Some(if let Some(modifier) = modifier { + format!("${{{index}:{modifier}}}") + } else { + format!("${index}") + }) + }) + .unwrap_or_else(|| { + self.error_logger.print_error_position( + node.token.as_ref().unwrap().pos, + "unsupported asm register modifier for target", + ); + unreachable!() + }); + let template = template + .bytes() + .map(|byte| format!("\\{byte:02X}")) + .collect::(); + let mut constraints = vec!["=r".to_string(); node.children.len()]; + constraints.extend(vec!["r".to_string(); node.args.len()]); + for (index, value) in matching { + constraints.push(index.to_string()); + arguments.push(value); + } + constraints.extend( + if cfg!(target_arch = "x86_64") { + vec!["~{dirflag}", "~{fpsr}", "~{flags}"] + } else { + vec!["~{cc}"] + } + .into_iter() + .map(String::from), + ); + if memory { + constraints.push("~{memory}".to_string()); + } + let types = node + .children + .iter() + .map(|output| Self::gen_type(&output.resolve_type_view().unwrap())) + .collect::>(); + let result_type = if types.is_empty() { + "void".to_string() + } else if types.len() == 1 { + types[0].clone() + } else { + format!("{{ {} }}", types.join(", ")) + }; + let result = if types.is_empty() { + String::new() + } else { + let register = format!("%{}", options.new_temporary()); + register + }; + let assignment = if types.is_empty() { + String::new() + } else { + format!("{result} = ") + }; + lines.push(format!( + " {assignment}call {result_type} asm {}\"{template}\", \"{}\"({})", + if volatile { "sideeffect " } else { "" }, + constraints.join(","), + arguments.join(", ") + )); + for (index, output) in node.children.iter().enumerate() { + let value = if types.len() == 1 { + result.clone() + } else { + let value = format!("%{}", options.new_temporary()); + lines.push(format!( + " {value} = extractvalue {result_type} {result}, {index}" + )); + value + }; + lines.extend(self.gen_addr(output, options)); + let address = options.register_queue.pop().unwrap(); + lines.push(format!( + " store {}{} {value}, ptr {address}, align {}", + if output.volatile_access { + "volatile " + } else { + "" + }, + types[index], + Self::access_alignment(output) + )); + } + options.register_queue.push("0".to_string()); + lines + } +} diff --git a/src/generator/x86_64/generator.rs b/src/generator/x86_64/generator.rs index f6fef30..14ee983 100644 --- a/src/generator/x86_64/generator.rs +++ b/src/generator/x86_64/generator.rs @@ -1,3 +1,8 @@ +mod aggregate; +mod atomic; +mod globals; +mod inline_asm; + use super::assembly::Assembly; use super::instruction::{ InstOperand::*, @@ -10,9 +15,7 @@ use crate::error; use crate::func::Func; use crate::generator::Os; use crate::generator::{ScalarLocation, scalar_arguments}; -use crate::global::{GlobalVariable, GlobalVariableData}; -use crate::node::{AtomicOperation, AtomicRmwOperation, Node, NodeType, render_inline_assembly}; -use crate::token::Token; +use crate::node::{Node, NodeType}; use std::{ collections::{HashMap, HashSet}, fmt::Display, @@ -148,264 +151,6 @@ impl<'a> AsmGenerator<'a> { ) } - fn gen_string_literals(&self, string_literals: &[Token]) -> Assembly { - if string_literals.is_empty() { - vec![] - } else { - vec![ - if let Os::MacOS = self.target_os { - ".section __TEXT,__const".into() - } else { - ".section .data".into() - }, - string_literals - .iter() - .enumerate() - .map(|(i, token)| { - let bytes = token.encoded_string(); - vec![ - format!(" .balign {}", token.string_encoding.unit_size()).into(), - format!("L_.str.{}:", i).into(), - format!( - " .byte {}", - bytes - .into_iter() - .map(|byte| byte.to_string()) - .collect::>() - .join(", ") - ) - .into(), - ] - .into() - }) - .collect::>() - .into(), - ] - } - .into() - } - - fn weak_reference(&self, name: &str) -> Assembly { - let directive = match self.target_os { - Os::MacOS => ".weak_reference", - Os::Linux => ".weak", - }; - format!("{directive} {}", self.with_prefix(name)).into() - } - - fn gen_global_variable(&self, name: &str, gv: &GlobalVariable) -> Assembly { - if gv.declaration_only { - return if gv.weak_import { - self.weak_reference(name) - } else { - vec![].into() - }; - } - if gv.common && !gv.internal && !gv.thread_local && gv.data.is_none() { - let alignment = gv.ty.align_of().max(gv.alignment).max(1); - let alignment = match self.target_os { - Os::MacOS => alignment.trailing_zeros() as usize, - Os::Linux => alignment, - }; - return format!( - ".comm {},{},{}", - self.with_prefix(name), - gv.ty.size_of(), - alignment - ) - .into(); - } - vec![ - if let Os::MacOS = self.target_os { - ".section __DATA,__data".into() - } else { - ".section .data".into() - }, - if gv.internal { - vec![].into() - } else { - format!(".globl {}", self.with_prefix(name)).into() - }, - format!(" .balign {}", gv.ty.align_of().max(gv.alignment)).into(), - format!("{}:", self.with_prefix(name)).into(), - self.gen_initializer_element(&gv.ty, gv.data.as_ref()), - ] - .into() - } - - fn gen_initializer_element(&self, ty: &Type, data: Option<&GlobalVariableData>) -> Assembly { - let data = if !matches!(ty, Type::Arr(..) | Type::Struct(_)) { - let mut value = data; - while let Some(GlobalVariableData::Arr(elements)) = value { - value = elements.first(); - } - value - } else { - data - }; - match ty { - Type::Aligned(child, _) => self.gen_initializer_element(child, data), - Type::Struct(struct_type) - if struct_type.has_bit_fields() || struct_type.definition().borrow().is_union => - { - let mut bytes = vec![0; ty.size_of()]; - let mut addresses = std::collections::HashMap::new(); - crate::global::initializer_bytes(ty, data, 0, &mut bytes, &mut addresses); - let mut lines = vec![]; - let mut index = 0; - while index < bytes.len() { - if let Some((symbol, delta)) = addresses.get(&index) { - let symbol = self.address_symbol(symbol); - lines.push(format!(" .8byte {symbol}{delta:+}").into()); - index += 8; - } else { - lines.push(format!(" .byte {}", bytes[index]).into()); - index += 1; - } - } - lines.into() - } - Type::Struct(struct_type) => { - let elements = if let Some(GlobalVariableData::Arr(elements)) = data { - Some(elements) - } else { - None - }; - let mut lines = vec![]; - let mut end = 0; - for (i, member) in struct_type - .members() - .iter() - .take(if struct_type.definition().borrow().is_union { - 1 - } else { - usize::MAX - }) - .enumerate() - { - if member.offset > end { - lines.push(format!(" .zero {}", member.offset - end).into()); - } - lines.push(self.gen_initializer_element( - &member.ty, - elements.and_then(|elements| elements.get(i)), - )); - end = member.offset + member.ty.size_of(); - } - if ty.size_of() > end { - lines.push(format!(" .zero {}", ty.size_of() - end).into()); - } - lines.into() - } - Type::Arr(children_ty, size) => { - let size = size.unwrap_or(0); - if let Some(GlobalVariableData::Arr(v)) = data { - vec![ - v.iter() - .enumerate() - .filter(|(i, _)| *i < size) - .map(|(_, d)| { - self.gen_initializer_element(children_ty.as_ref(), Some(d)) - }) - .collect::>() - .into(), - format!( - " .zero {}", - children_ty.size_of() - * v.iter() - .enumerate() - .filter(|(i, _)| *i < size) - .fold(size, |_, _| { size.saturating_sub(v.len()) }) - ) - .into(), - ] - .into() - } else { - format!(" .zero {}", children_ty.size_of() * size).into() - } - } - Type::Enum(value) => self.gen_initializer_element(&value.underlying.borrow(), data), - Type::I8 | Type::SignedChar => format!( - " .byte {}", - if let Some(GlobalVariableData::Elem(s)) = data { - s - } else { - "0" - } - ) - .into(), - Type::Integer128 { .. } => " .zero 16".into(), - Type::Float16 => " .zero 2".into(), - Type::Vector(..) => format!(" .zero {}", ty.size_of()).into(), - Type::LongDouble(bits) if *bits != 64 => { - let value = if let Some(GlobalVariableData::Elem(value)) = data { - value.parse::().unwrap() - } else { - 0 - }; - format!( - " .8byte {}\n .8byte {}", - value as u64, - (value >> 64) as u64 - ) - .into() - } - Type::Float(_) | Type::LongDouble(_) => format!( - " .{}byte {}", - ty.size_of(), - if let Some(GlobalVariableData::Elem(value)) = data { - value.as_str() - } else { - "0" - } - ) - .into(), - Type::Integer { .. } | Type::LongLong { .. } | Type::Bool => format!( - " {} {}", - if ty.size_of() == 1 { - ".byte".to_string() - } else { - format!(".{}byte", ty.size_of()) - }, - if let Some(GlobalVariableData::Address { symbol, offset }) = data { - format!("{}{offset:+}", self.address_symbol(symbol)) - } else if let Some(GlobalVariableData::Elem(value)) = data { - ty.convert_constant(value.parse().unwrap()).to_string() - } else { - "0".into() - } - ) - .into(), - Type::I32 => format!( - " .4byte {}", - if let Some(GlobalVariableData::Elem(s)) = data { - s - } else { - "0" - } - ) - .into(), - _ => format!( - " .8byte {}", - if let Some(GlobalVariableData::Address { symbol, offset }) = data { - let symbol = self.address_symbol(symbol); - format!("{symbol}{offset:+}") - } else if let Some(GlobalVariableData::Elem(s)) = data { - if s.starts_with("@.str.") { - s.replace('@', "L_") - } else if let Some(name) = s.strip_prefix('@') { - self.address_symbol(name) - } else { - s.clone() - } - } else { - "0".to_string() - } - ) - .into(), - } - } - fn gen_func(&self, name: &str, func: &Func, offset: usize) -> Assembly { if func.body.is_some() && func @@ -770,501 +515,6 @@ impl<'a> AsmGenerator<'a> { lines.push(Assembly::inst1(PUSH, RAX)); lines.into() } - fn aggregate_load(offset: usize, size: usize) -> Assembly { - let mut lines = vec![Assembly::inst2(MOV, R11, 0)]; - for index in (0..size).rev() { - lines.extend([ - Assembly::inst2(SHL, R11, 8), - Assembly::inst2(MOVZX, RAX, PtrOffset(R10, offset + index, 1)), - Assembly::inst2(OR, R11, RAX), - ]); - } - lines.push(Assembly::inst2(MOV, RAX, R11)); - lines.into() - } - fn aggregate_store(size: usize) -> Assembly { - let mut lines = vec![]; - for index in 0..size { - lines.push(Assembly::inst2(MOV, PtrOffset(R11, index, 1), AL)); - if index + 1 < size { - lines.push(Assembly::inst2(SHR, RAX, 8)); - } - } - lines.into() - } - fn aggregate_copy(size: usize) -> Assembly { - (0..size) - .map(|index| { - vec![ - Assembly::inst2(MOVZX, RAX, PtrOffset(R10, index, 1)), - Assembly::inst2(MOV, PtrOffset(R11, index, 1), AL), - ] - .into() - }) - .collect::>() - .into() - } - fn aggregate_function(&self, name: &str, func: &Func, locals: usize) -> Assembly { - use crate::generator::abi::{self, Location}; - let (types, result, variadic) = func.cty.function_signature().unwrap(); - let signature = abi::signature( - &types, - &result, - crate::generator::Arch::X86_64, - self.target_os, - types.len(), - variadic, - ); - let offset = locals + if variadic { 192 } else { 128 }; - let mut lines = vec![]; - if !func.internal { - lines.push(format!(".globl {}", self.with_prefix(name)).into()); - } - lines.extend([ - format!("{}:", self.with_prefix(name)).into(), - Assembly::inst1(PUSH, RBP), - Assembly::inst2(MOV, RBP, RSP), - Assembly::inst2(SUB, RSP, offset), - ]); - for (index, register) in ARGS_REG.iter().enumerate() { - lines.push(Assembly::inst2( - MOV, - PtrOffset(RSP, index * 8, 8), - *register, - )); - } - for index in 0..8 { - lines.push(Assembly::inst2( - MOVQ, - PtrOffset(RSP, 48 + index * if variadic { 16 } else { 8 }, 8), - Xmm(index), - )); - } - if signature.indirect_result { - lines.push(Assembly::inst2(MOV, RAX, PtrOffset(RSP, 0, 8))); - lines.extend([ - Assembly::inst2(MOV, R11, RBP), - Assembly::inst2(SUB, R11, func.return_pointer.unwrap()), - Assembly::inst2(MOV, Ptr(R11, 8), RAX), - ]); - } - for (index, argument) in signature.arguments.iter().enumerate() { - let destination = func.args[index].offset.unwrap(); - if let Some(stack) = argument.memory { - lines.extend([ - Assembly::inst2(LEA, R10, PtrOffset(RBP, 16 + stack, 8)), - Assembly::inst2(MOV, R11, RBP), - Assembly::inst2(SUB, R11, destination), - Self::aggregate_copy(types[index].size_of()), - ]); - } else { - for (part, location) in &argument.parts { - match *location { - Location::Stack(stack) => { - lines.push(Assembly::inst2(MOV, RAX, PtrOffset(RBP, 16 + stack, 8))) - } - location => { - let slot = match location { - Location::General(n) => n * 8, - Location::Floating(n) => 48 + n * if variadic { 16 } else { 8 }, - _ => unreachable!(), - }; - lines.extend([ - Assembly::inst2(MOV, R10, RBP), - Assembly::inst2(SUB, R10, offset - slot), - Assembly::inst2(MOV, RAX, Ptr(R10, 8)), - ]); - } - } - lines.extend([ - Assembly::inst2(MOV, R11, RBP), - Assembly::inst2(SUB, R11, destination), - Assembly::inst2(ADD, R11, part.offset), - Self::aggregate_store( - (types[index].size_of() - part.offset).min(part.ty.size_of()), - ), - ]); - } - } - } - lines.extend([ - self.gen_node(func.body.as_ref().unwrap(), offset, 0), - Assembly::inst2(MOV, RAX, 0), - Assembly::epilogue(), - ]); - lines.into() - } - fn aggregate_call(&self, node: &Node, offset: usize, branch: usize) -> Assembly { - use crate::generator::abi::{self, Location}; - let types = node - .args - .iter() - .rev() - .map(|arg| arg.resolve_type().unwrap()) - .collect::>(); - let result = node.resolve_type().unwrap(); - let signature = abi::signature( - &types, - &result, - crate::generator::Arch::X86_64, - self.target_os, - node.parameter_types.len(), - node.variadic, - ); - let mut lines = vec![]; - if let Some(callee) = &node.lhs { - lines.push(self.gen_node(callee, offset, branch)); - } - for arg in &node.args { - lines.push(self.gen_node(arg, offset, branch)); - } - lines.extend([ - Assembly::inst2(MOV, R11, RSP), - Assembly::inst2(SUB, RSP, signature.stack_size + 8), - Assembly::inst2(AND, RSP, "-16"), - Assembly::inst2(MOV, PtrOffset(RSP, signature.stack_size, 8), R11), - ]); - // Copy stack aggregates before loading argument registers. - for (index, argument) in signature.arguments.iter().enumerate() { - if let Some(stack) = argument.memory { - lines.extend([ - Assembly::inst2(MOV, R10, PtrOffset(RSP, signature.stack_size, 8)), - Assembly::inst2(MOV, R10, PtrOffset(R10, index * 8, 8)), - Assembly::inst2(LEA, R11, PtrOffset(RSP, stack, 8)), - Self::aggregate_copy(types[index].size_of()), - ]); - } - } - for (index, argument) in signature.arguments.iter().enumerate() { - if argument.memory.is_some() { - continue; - } - for (part, location) in &argument.parts { - lines.push(Assembly::inst2( - MOV, - R10, - PtrOffset(RSP, signature.stack_size, 8), - )); - if matches!(types[index], Type::Struct(_)) { - lines.extend([ - Assembly::inst2(MOV, R10, PtrOffset(R10, index * 8, 8)), - Self::aggregate_load( - part.offset, - (types[index].size_of() - part.offset).min(part.ty.size_of()), - ), - ]); - } else { - lines.push(Assembly::inst2(MOV, RAX, PtrOffset(R10, index * 8, 8))); - } - lines.push(match *location { - Location::General(register) => Assembly::inst2(MOV, ARGS_REG[register], RAX), - Location::Floating(register) => Assembly::inst2( - if part.ty.float_bits() == Some(32) { - MOVD - } else { - MOVQ - }, - Xmm(register), - if part.ty.float_bits() == Some(32) { - EAX - } else { - RAX - }, - ), - Location::Stack(stack) => Assembly::inst2(MOV, PtrOffset(RSP, stack, 8), RAX), - }); - } - } - if signature.indirect_result { - lines.extend([ - Assembly::inst2(MOV, RDI, RBP), - Assembly::inst2(SUB, RDI, node.offset.unwrap()), - ]); - } - if node.lhs.is_some() { - lines.extend([ - Assembly::inst2(MOV, R10, PtrOffset(RSP, signature.stack_size, 8)), - Assembly::inst2(MOV, R10, PtrOffset(R10, node.args.len() * 8, 8)), - ]); - } - lines.push(Assembly::inst2(MOV, RAX, signature.floating)); - lines.push(if node.lhs.is_some() { - Assembly::inst1(CALL, R10) - } else { - Assembly::inst1(CALL, self.with_prefix(&node.global_name)) - }); - lines.extend([ - Assembly::inst2(MOV, RSP, PtrOffset(RSP, signature.stack_size, 8)), - Assembly::inst2( - ADD, - RSP, - (node.args.len() + usize::from(node.lhs.is_some())) * 8, - ), - ]); - if matches!(result, Type::Struct(_)) { - if !signature.indirect_result { - lines.extend([Assembly::inst1(PUSH, RAX), Assembly::inst1(PUSH, RDX)]); - let (mut gp, mut fp) = (0, 0); - for part in &signature.result { - if part.ty.float_bits().is_some() { - lines.push(Assembly::inst2( - if part.ty.float_bits() == Some(32) { - MOVD - } else { - MOVQ - }, - if part.ty.float_bits() == Some(32) { - EAX - } else { - RAX - }, - Xmm(fp), - )); - fp += 1; - } else { - lines.push(Assembly::inst2( - MOV, - RAX, - PtrOffset(RSP, if gp == 0 { 8 } else { 0 }, 8), - )); - gp += 1; - } - lines.extend([ - Assembly::inst2(MOV, R11, RBP), - Assembly::inst2(SUB, R11, node.offset.unwrap()), - Assembly::inst2(ADD, R11, part.offset), - Self::aggregate_store( - (result.size_of() - part.offset).min(part.ty.size_of()), - ), - ]); - } - lines.push(Assembly::inst2(ADD, RSP, 16)); - } - lines.extend([ - Assembly::inst2(MOV, RAX, RBP), - Assembly::inst2(SUB, RAX, node.offset.unwrap()), - ]); - } else if let Some(bits) = result.float_bits() { - lines.push(Assembly::inst2( - if bits == 32 { MOVD } else { MOVQ }, - if bits == 32 { EAX } else { RAX }, - Xmm(0), - )); - } else { - lines.push(Self::normalize_integer(&result, RAX)); - } - lines.push(Assembly::inst1(PUSH, RAX)); - lines.into() - } - fn aggregate_return(&self, node: &Node, offset: usize, branch: usize) -> Assembly { - use crate::generator::abi; - let ty = node.resolve_type().unwrap(); - let parts = abi::aggregate_parts(&ty, &crate::generator::Arch::X86_64); - let mut lines = vec![ - self.gen_node(node.lhs.as_ref().unwrap(), offset, branch), - Assembly::inst1(POP, R10), - ]; - if parts.is_empty() { - lines.extend([ - Assembly::inst2(MOV, R11, RBP), - Assembly::inst2(SUB, R11, node.offset.unwrap()), - Assembly::inst2(MOV, R11, Ptr(R11, 8)), - Self::aggregate_copy(ty.size_of()), - Assembly::inst2(MOV, RAX, R11), - ]); - } else { - let (mut gp, mut fp) = (0, 0); - let mut outputs = vec![]; - for part in parts { - let register = if part.ty.float_bits().is_some() { - let n = fp; - fp += 1; - Xmm(n) - } else { - let r = if gp == 0 { RAX } else { RDX }; - gp += 1; - r - }; - outputs.push((part, register)); - } - // Load RAX last because it is also the byte-assembly scratch register. - outputs.sort_by_key(|(_, register)| matches!(register, RAX)); - for (part, register) in &outputs { - lines.push(Self::aggregate_load( - part.offset, - (ty.size_of() - part.offset).min(part.ty.size_of()), - )); - lines.push(Assembly::inst2( - if part.ty.float_bits() == Some(32) { - MOVD - } else if part.ty.float_bits().is_some() { - MOVQ - } else { - MOV - }, - *register, - if part.ty.float_bits() == Some(32) { - EAX - } else { - RAX - }, - )); - } - } - lines.push(Assembly::epilogue()); - lines.into() - } - fn atomic_update(node: &Node, operation: AtomicRmwOperation, return_new: bool) -> Assembly { - let ty = node.args[0].dest_type().unwrap(); - let size = ty.size_of(); - let label = format!(".Latomic.{}", node.token.as_ref().unwrap().pos); - let mut lines = vec![ - Assembly::inst2(MOV, RSI, RAX), - if size == 8 { - Assembly::inst2(MOV, RAX, Ptr(RDI, 8)) - } else if size == 4 { - Assembly::inst2(MOV, EAX, Ptr(RDI, 4)) - } else { - Assembly::inst2(MOVZX, RAX, Ptr(RDI, size)) - }, - format!("{label}:").into(), - Assembly::inst2(MOV, RCX, RAX), - ]; - lines.push(match operation { - AtomicRmwOperation::Exchange => Assembly::inst2(MOV, RCX, RSI), - AtomicRmwOperation::Nand => { - vec![Assembly::inst2(AND, RCX, RSI), Assembly::inst1(NOT, RCX)].into() - } - _ => Assembly::inst2( - match operation { - AtomicRmwOperation::Add => ADD, - AtomicRmwOperation::Sub => SUB, - AtomicRmwOperation::And => AND, - AtomicRmwOperation::Or => OR, - AtomicRmwOperation::Xor => XOR, - _ => unreachable!(), - }, - RCX, - RSI, - ), - }); - lines.extend([ - Assembly::inst2( - CMPXCHG, - Ptr(RDI, size), - match size { - 1 => CL, - 2 => CX, - 4 => ECX, - _ => RCX, - }, - ), - Assembly::inst1(JNE, label), - ]); - if return_new { - lines.push(Assembly::inst2(MOV, RAX, RCX)); - } - lines.extend([ - Self::normalize_integer(&ty, RAX), - Assembly::inst1(PUSH, RAX), - ]); - lines.into() - } - - fn atomic_compare_exchange(node: &Node) -> Assembly { - let size = node.args[0].dest_type().unwrap().size_of(); - let done = format!(".Latomic.cas.done.{}", node.token.as_ref().unwrap().pos); - vec![ - Assembly::inst1(POP, R11), - Assembly::inst1(POP, R11), - Assembly::inst1(POP, R11), - Assembly::inst1(POP, RCX), - Assembly::inst1(POP, RDX), - Assembly::inst1(POP, RDI), - if size == 8 { - Assembly::inst2(MOV, RAX, Ptr(RDX, 8)) - } else if size == 4 { - Assembly::inst2(MOV, EAX, Ptr(RDX, 4)) - } else { - Assembly::inst2(MOVZX, RAX, Ptr(RDX, size)) - }, - Assembly::inst2( - CMPXCHG, - Ptr(RDI, size), - match size { - 1 => CL, - 2 => CX, - 4 => ECX, - _ => RCX, - }, - ), - Assembly::inst1(JE, done.clone()), - Assembly::inst2( - MOV, - Ptr(RDX, size), - match size { - 1 => AL, - 2 => AX, - 4 => EAX, - _ => RAX, - }, - ), - format!("{done}:").into(), - Assembly::inst1(SETE, AL), - Assembly::inst2(MOVZX, RAX, AL), - Assembly::inst1(PUSH, RAX), - ] - .into() - } - - fn gen_atomic( - &self, - node: &Node, - operation: AtomicOperation, - offset: usize, - breakable_branch_num: usize, - ) -> Assembly { - let mut lines = vec![]; - for argument in &node.args { - lines.push(self.gen_node(argument, offset, breakable_branch_num)); - } - if matches!(operation, AtomicOperation::CompareExchange(_)) { - lines.push(Self::atomic_compare_exchange(node)); - return lines.into(); - } - lines.push(Assembly::inst1(POP, R11)); - if operation == AtomicOperation::Load { - lines.extend([ - Assembly::inst1(POP, RAX), - Assembly::inst0(MFENCE), - self.deref_rax(node), - Assembly::inst0(MFENCE), - Self::normalize_integer(&node.resolve_type().unwrap(), RAX), - Assembly::inst1(PUSH, RAX), - ]); - } else { - let size = node.args[0].dest_type().unwrap().size_of(); - lines.extend([Assembly::inst1(POP, RAX), Assembly::inst1(POP, RDI)]); - if let AtomicOperation::Update(op, return_new) = operation { - lines.push(Self::atomic_update(node, op, return_new)); - } else { - lines.extend([ - Assembly::inst2( - XCHG, - Ptr(RDI, size), - match size { - 1 => AL, - 2 => AX, - 4 => EAX, - _ => RAX, - }, - ), - Assembly::inst1(PUSH, 0), - ]); - } - } - lines.into() - } fn gen_prefetch( &self, @@ -1290,83 +540,6 @@ impl<'a> AsmGenerator<'a> { .into() } - fn gen_inline_assembly( - &self, - node: &Node, - offset: usize, - breakable_branch_num: usize, - ) -> Assembly { - let registers = [RAX, R10, R11, RCX, RDX, RSI, RDI]; - let names = [ - ["%al", "%ax", "%eax", "%rax"], - ["%r10b", "%r10w", "%r10d", "%r10"], - ["%r11b", "%r11w", "%r11d", "%r11"], - ["%cl", "%cx", "%ecx", "%rcx"], - ["%dl", "%dx", "%edx", "%rdx"], - ["%sil", "%si", "%esi", "%rsi"], - ["%dil", "%di", "%edi", "%rdi"], - ]; - let operands = node.children.iter().chain(&node.args).collect::>(); - let template = render_inline_assembly(&node.dest, |index, modifier| { - let natural = operands.get(index)?.resolve_type()?.size_of(); - let width = crate::generator::asm_register_width( - crate::generator::Arch::X86_64, - modifier, - natural, - )?; - Some(names.get(index)?[width.trailing_zeros() as usize].into()) - }) - .unwrap_or_else(|| { - self.error_logger.print_error_position( - node.token.as_ref().unwrap().pos, - "unsupported asm register modifier for x86_64", - ); - unreachable!() - }); - let mut lines = vec![]; - let matching = node - .children - .iter() - .enumerate() - .filter(|(index, _)| node.value.unwrap() & (1 << index) != 0) - .collect::>(); - for (_, input) in &matching { - lines.push(self.gen_node(input, offset, breakable_branch_num)); - } - for input in &node.args { - lines.push(self.gen_node(input, offset, breakable_branch_num)); - } - for (index, input) in node - .args - .iter() - .enumerate() - .rev() - .map(|(index, input)| (index + node.children.len(), input)) - .chain(matching.into_iter().rev()) - { - let register = registers[index]; - lines.push(Assembly::inst1(POP, register)); - if input.resolve_type().unwrap().size_of() == 4 { - lines.push(Self::normalize_integer(&Type::integer(32, false), register)); - } - } - lines.push(Assembly::inst1(InlineAsm, InlineAsmTemplate(template))); - // Buffer every result before address generation can overwrite operand registers. - for register in registers[..node.children.len()].iter().rev() { - lines.push(Assembly::inst1(PUSH, *register)); - } - for output in &node.children { - lines.push(self.gen_addr(output, offset, breakable_branch_num)); - lines.extend([ - Assembly::inst1(POP, RAX), - Assembly::inst1(POP, RDI), - self.operation2rdi(output.resolve_type(), MOV, RAX), - ]); - } - lines.push(Assembly::inst1(PUSH, 0)); - lines.into() - } - fn gen_node(&self, node: &Node, offset: usize, breakable_branch_num: usize) -> Assembly { if node .resolve_type_view() diff --git a/src/generator/x86_64/generator/aggregate.rs b/src/generator/x86_64/generator/aggregate.rs new file mode 100644 index 0000000..6f2f220 --- /dev/null +++ b/src/generator/x86_64/generator/aggregate.rs @@ -0,0 +1,359 @@ +use super::ARGS_REG; +use super::AsmGenerator; +use crate::ctype::Type; +use crate::func::Func; +use crate::generator::x86_64::assembly::Assembly; +use crate::generator::x86_64::instruction::{InstOperand::*, InstOperator::*, Register::*}; +use crate::node::Node; + +impl AsmGenerator<'_> { + pub(super) fn aggregate_load(offset: usize, size: usize) -> Assembly { + let mut lines = vec![Assembly::inst2(MOV, R11, 0)]; + for index in (0..size).rev() { + lines.extend([ + Assembly::inst2(SHL, R11, 8), + Assembly::inst2(MOVZX, RAX, PtrOffset(R10, offset + index, 1)), + Assembly::inst2(OR, R11, RAX), + ]); + } + lines.push(Assembly::inst2(MOV, RAX, R11)); + lines.into() + } + + pub(super) fn aggregate_store(size: usize) -> Assembly { + let mut lines = vec![]; + for index in 0..size { + lines.push(Assembly::inst2(MOV, PtrOffset(R11, index, 1), AL)); + if index + 1 < size { + lines.push(Assembly::inst2(SHR, RAX, 8)); + } + } + lines.into() + } + + pub(super) fn aggregate_copy(size: usize) -> Assembly { + (0..size) + .map(|index| { + vec![ + Assembly::inst2(MOVZX, RAX, PtrOffset(R10, index, 1)), + Assembly::inst2(MOV, PtrOffset(R11, index, 1), AL), + ] + .into() + }) + .collect::>() + .into() + } + + pub(super) fn aggregate_function(&self, name: &str, func: &Func, locals: usize) -> Assembly { + use crate::generator::abi::{self, Location}; + let (types, result, variadic) = func.cty.function_signature().unwrap(); + let signature = abi::signature( + &types, + &result, + crate::generator::Arch::X86_64, + self.target_os, + types.len(), + variadic, + ); + let offset = locals + if variadic { 192 } else { 128 }; + let mut lines = vec![]; + if !func.internal { + lines.push(format!(".globl {}", self.with_prefix(name)).into()); + } + lines.extend([ + format!("{}:", self.with_prefix(name)).into(), + Assembly::inst1(PUSH, RBP), + Assembly::inst2(MOV, RBP, RSP), + Assembly::inst2(SUB, RSP, offset), + ]); + for (index, register) in ARGS_REG.iter().enumerate() { + lines.push(Assembly::inst2( + MOV, + PtrOffset(RSP, index * 8, 8), + *register, + )); + } + for index in 0..8 { + lines.push(Assembly::inst2( + MOVQ, + PtrOffset(RSP, 48 + index * if variadic { 16 } else { 8 }, 8), + Xmm(index), + )); + } + if signature.indirect_result { + lines.push(Assembly::inst2(MOV, RAX, PtrOffset(RSP, 0, 8))); + lines.extend([ + Assembly::inst2(MOV, R11, RBP), + Assembly::inst2(SUB, R11, func.return_pointer.unwrap()), + Assembly::inst2(MOV, Ptr(R11, 8), RAX), + ]); + } + for (index, argument) in signature.arguments.iter().enumerate() { + let destination = func.args[index].offset.unwrap(); + if let Some(stack) = argument.memory { + lines.extend([ + Assembly::inst2(LEA, R10, PtrOffset(RBP, 16 + stack, 8)), + Assembly::inst2(MOV, R11, RBP), + Assembly::inst2(SUB, R11, destination), + Self::aggregate_copy(types[index].size_of()), + ]); + } else { + for (part, location) in &argument.parts { + match *location { + Location::Stack(stack) => { + lines.push(Assembly::inst2(MOV, RAX, PtrOffset(RBP, 16 + stack, 8))) + } + location => { + let slot = match location { + Location::General(n) => n * 8, + Location::Floating(n) => 48 + n * if variadic { 16 } else { 8 }, + _ => unreachable!(), + }; + lines.extend([ + Assembly::inst2(MOV, R10, RBP), + Assembly::inst2(SUB, R10, offset - slot), + Assembly::inst2(MOV, RAX, Ptr(R10, 8)), + ]); + } + } + lines.extend([ + Assembly::inst2(MOV, R11, RBP), + Assembly::inst2(SUB, R11, destination), + Assembly::inst2(ADD, R11, part.offset), + Self::aggregate_store( + (types[index].size_of() - part.offset).min(part.ty.size_of()), + ), + ]); + } + } + } + lines.extend([ + self.gen_node(func.body.as_ref().unwrap(), offset, 0), + Assembly::inst2(MOV, RAX, 0), + Assembly::epilogue(), + ]); + lines.into() + } + + pub(super) fn aggregate_call(&self, node: &Node, offset: usize, branch: usize) -> Assembly { + use crate::generator::abi::{self, Location}; + let types = node + .args + .iter() + .rev() + .map(|arg| arg.resolve_type().unwrap()) + .collect::>(); + let result = node.resolve_type().unwrap(); + let signature = abi::signature( + &types, + &result, + crate::generator::Arch::X86_64, + self.target_os, + node.parameter_types.len(), + node.variadic, + ); + let mut lines = vec![]; + if let Some(callee) = &node.lhs { + lines.push(self.gen_node(callee, offset, branch)); + } + for arg in &node.args { + lines.push(self.gen_node(arg, offset, branch)); + } + lines.extend([ + Assembly::inst2(MOV, R11, RSP), + Assembly::inst2(SUB, RSP, signature.stack_size + 8), + Assembly::inst2(AND, RSP, "-16"), + Assembly::inst2(MOV, PtrOffset(RSP, signature.stack_size, 8), R11), + ]); + // Copy stack aggregates before loading argument registers. + for (index, argument) in signature.arguments.iter().enumerate() { + if let Some(stack) = argument.memory { + lines.extend([ + Assembly::inst2(MOV, R10, PtrOffset(RSP, signature.stack_size, 8)), + Assembly::inst2(MOV, R10, PtrOffset(R10, index * 8, 8)), + Assembly::inst2(LEA, R11, PtrOffset(RSP, stack, 8)), + Self::aggregate_copy(types[index].size_of()), + ]); + } + } + for (index, argument) in signature.arguments.iter().enumerate() { + if argument.memory.is_some() { + continue; + } + for (part, location) in &argument.parts { + lines.push(Assembly::inst2( + MOV, + R10, + PtrOffset(RSP, signature.stack_size, 8), + )); + if matches!(types[index], Type::Struct(_)) { + lines.extend([ + Assembly::inst2(MOV, R10, PtrOffset(R10, index * 8, 8)), + Self::aggregate_load( + part.offset, + (types[index].size_of() - part.offset).min(part.ty.size_of()), + ), + ]); + } else { + lines.push(Assembly::inst2(MOV, RAX, PtrOffset(R10, index * 8, 8))); + } + lines.push(match *location { + Location::General(register) => Assembly::inst2(MOV, ARGS_REG[register], RAX), + Location::Floating(register) => Assembly::inst2( + if part.ty.float_bits() == Some(32) { + MOVD + } else { + MOVQ + }, + Xmm(register), + if part.ty.float_bits() == Some(32) { + EAX + } else { + RAX + }, + ), + Location::Stack(stack) => Assembly::inst2(MOV, PtrOffset(RSP, stack, 8), RAX), + }); + } + } + if signature.indirect_result { + lines.extend([ + Assembly::inst2(MOV, RDI, RBP), + Assembly::inst2(SUB, RDI, node.offset.unwrap()), + ]); + } + if node.lhs.is_some() { + lines.extend([ + Assembly::inst2(MOV, R10, PtrOffset(RSP, signature.stack_size, 8)), + Assembly::inst2(MOV, R10, PtrOffset(R10, node.args.len() * 8, 8)), + ]); + } + lines.push(Assembly::inst2(MOV, RAX, signature.floating)); + lines.push(if node.lhs.is_some() { + Assembly::inst1(CALL, R10) + } else { + Assembly::inst1(CALL, self.with_prefix(&node.global_name)) + }); + lines.extend([ + Assembly::inst2(MOV, RSP, PtrOffset(RSP, signature.stack_size, 8)), + Assembly::inst2( + ADD, + RSP, + (node.args.len() + usize::from(node.lhs.is_some())) * 8, + ), + ]); + if matches!(result, Type::Struct(_)) { + if !signature.indirect_result { + lines.extend([Assembly::inst1(PUSH, RAX), Assembly::inst1(PUSH, RDX)]); + let (mut gp, mut fp) = (0, 0); + for part in &signature.result { + if part.ty.float_bits().is_some() { + lines.push(Assembly::inst2( + if part.ty.float_bits() == Some(32) { + MOVD + } else { + MOVQ + }, + if part.ty.float_bits() == Some(32) { + EAX + } else { + RAX + }, + Xmm(fp), + )); + fp += 1; + } else { + lines.push(Assembly::inst2( + MOV, + RAX, + PtrOffset(RSP, if gp == 0 { 8 } else { 0 }, 8), + )); + gp += 1; + } + lines.extend([ + Assembly::inst2(MOV, R11, RBP), + Assembly::inst2(SUB, R11, node.offset.unwrap()), + Assembly::inst2(ADD, R11, part.offset), + Self::aggregate_store( + (result.size_of() - part.offset).min(part.ty.size_of()), + ), + ]); + } + lines.push(Assembly::inst2(ADD, RSP, 16)); + } + lines.extend([ + Assembly::inst2(MOV, RAX, RBP), + Assembly::inst2(SUB, RAX, node.offset.unwrap()), + ]); + } else if let Some(bits) = result.float_bits() { + lines.push(Assembly::inst2( + if bits == 32 { MOVD } else { MOVQ }, + if bits == 32 { EAX } else { RAX }, + Xmm(0), + )); + } else { + lines.push(Self::normalize_integer(&result, RAX)); + } + lines.push(Assembly::inst1(PUSH, RAX)); + lines.into() + } + + pub(super) fn aggregate_return(&self, node: &Node, offset: usize, branch: usize) -> Assembly { + use crate::generator::abi; + let ty = node.resolve_type().unwrap(); + let parts = abi::aggregate_parts(&ty, &crate::generator::Arch::X86_64); + let mut lines = vec![ + self.gen_node(node.lhs.as_ref().unwrap(), offset, branch), + Assembly::inst1(POP, R10), + ]; + if parts.is_empty() { + lines.extend([ + Assembly::inst2(MOV, R11, RBP), + Assembly::inst2(SUB, R11, node.offset.unwrap()), + Assembly::inst2(MOV, R11, Ptr(R11, 8)), + Self::aggregate_copy(ty.size_of()), + Assembly::inst2(MOV, RAX, R11), + ]); + } else { + let (mut gp, mut fp) = (0, 0); + let mut outputs = vec![]; + for part in parts { + let register = if part.ty.float_bits().is_some() { + let n = fp; + fp += 1; + Xmm(n) + } else { + let r = if gp == 0 { RAX } else { RDX }; + gp += 1; + r + }; + outputs.push((part, register)); + } + // Load RAX last because it is also the byte-assembly scratch register. + outputs.sort_by_key(|(_, register)| matches!(register, RAX)); + for (part, register) in &outputs { + lines.push(Self::aggregate_load( + part.offset, + (ty.size_of() - part.offset).min(part.ty.size_of()), + )); + lines.push(Assembly::inst2( + if part.ty.float_bits() == Some(32) { + MOVD + } else if part.ty.float_bits().is_some() { + MOVQ + } else { + MOV + }, + *register, + if part.ty.float_bits() == Some(32) { + EAX + } else { + RAX + }, + )); + } + } + lines.push(Assembly::epilogue()); + lines.into() + } +} diff --git a/src/generator/x86_64/generator/atomic.rs b/src/generator/x86_64/generator/atomic.rs new file mode 100644 index 0000000..bb9e3e6 --- /dev/null +++ b/src/generator/x86_64/generator/atomic.rs @@ -0,0 +1,162 @@ +use super::AsmGenerator; +use crate::generator::x86_64::assembly::Assembly; +use crate::generator::x86_64::instruction::{InstOperand::*, InstOperator::*, Register::*}; +use crate::node::{AtomicOperation, AtomicRmwOperation, Node}; + +impl AsmGenerator<'_> { + pub(super) fn atomic_update( + node: &Node, + operation: AtomicRmwOperation, + return_new: bool, + ) -> Assembly { + let ty = node.args[0].dest_type().unwrap(); + let size = ty.size_of(); + let label = format!(".Latomic.{}", node.token.as_ref().unwrap().pos); + let mut lines = vec![ + Assembly::inst2(MOV, RSI, RAX), + if size == 8 { + Assembly::inst2(MOV, RAX, Ptr(RDI, 8)) + } else if size == 4 { + Assembly::inst2(MOV, EAX, Ptr(RDI, 4)) + } else { + Assembly::inst2(MOVZX, RAX, Ptr(RDI, size)) + }, + format!("{label}:").into(), + Assembly::inst2(MOV, RCX, RAX), + ]; + lines.push(match operation { + AtomicRmwOperation::Exchange => Assembly::inst2(MOV, RCX, RSI), + AtomicRmwOperation::Nand => { + vec![Assembly::inst2(AND, RCX, RSI), Assembly::inst1(NOT, RCX)].into() + } + _ => Assembly::inst2( + match operation { + AtomicRmwOperation::Add => ADD, + AtomicRmwOperation::Sub => SUB, + AtomicRmwOperation::And => AND, + AtomicRmwOperation::Or => OR, + AtomicRmwOperation::Xor => XOR, + _ => unreachable!(), + }, + RCX, + RSI, + ), + }); + lines.extend([ + Assembly::inst2( + CMPXCHG, + Ptr(RDI, size), + match size { + 1 => CL, + 2 => CX, + 4 => ECX, + _ => RCX, + }, + ), + Assembly::inst1(JNE, label), + ]); + if return_new { + lines.push(Assembly::inst2(MOV, RAX, RCX)); + } + lines.extend([ + Self::normalize_integer(&ty, RAX), + Assembly::inst1(PUSH, RAX), + ]); + lines.into() + } + + pub(super) fn atomic_compare_exchange(node: &Node) -> Assembly { + let size = node.args[0].dest_type().unwrap().size_of(); + let done = format!(".Latomic.cas.done.{}", node.token.as_ref().unwrap().pos); + vec![ + Assembly::inst1(POP, R11), + Assembly::inst1(POP, R11), + Assembly::inst1(POP, R11), + Assembly::inst1(POP, RCX), + Assembly::inst1(POP, RDX), + Assembly::inst1(POP, RDI), + if size == 8 { + Assembly::inst2(MOV, RAX, Ptr(RDX, 8)) + } else if size == 4 { + Assembly::inst2(MOV, EAX, Ptr(RDX, 4)) + } else { + Assembly::inst2(MOVZX, RAX, Ptr(RDX, size)) + }, + Assembly::inst2( + CMPXCHG, + Ptr(RDI, size), + match size { + 1 => CL, + 2 => CX, + 4 => ECX, + _ => RCX, + }, + ), + Assembly::inst1(JE, done.clone()), + Assembly::inst2( + MOV, + Ptr(RDX, size), + match size { + 1 => AL, + 2 => AX, + 4 => EAX, + _ => RAX, + }, + ), + format!("{done}:").into(), + Assembly::inst1(SETE, AL), + Assembly::inst2(MOVZX, RAX, AL), + Assembly::inst1(PUSH, RAX), + ] + .into() + } + + pub(super) fn gen_atomic( + &self, + node: &Node, + operation: AtomicOperation, + offset: usize, + breakable_branch_num: usize, + ) -> Assembly { + let mut lines = vec![]; + for argument in &node.args { + lines.push(self.gen_node(argument, offset, breakable_branch_num)); + } + if matches!(operation, AtomicOperation::CompareExchange(_)) { + lines.push(Self::atomic_compare_exchange(node)); + return lines.into(); + } + lines.push(Assembly::inst1(POP, R11)); + if operation == AtomicOperation::Load { + lines.extend([ + Assembly::inst1(POP, RAX), + Assembly::inst0(MFENCE), + self.deref_rax(node), + Assembly::inst0(MFENCE), + Self::normalize_integer(&node.resolve_type().unwrap(), RAX), + Assembly::inst1(PUSH, RAX), + ]); + } else { + let size = node.args[0].dest_type().unwrap().size_of(); + lines.extend([Assembly::inst1(POP, RAX), Assembly::inst1(POP, RDI)]); + if let AtomicOperation::Update(op, return_new) = operation { + lines.push(Self::atomic_update(node, op, return_new)); + } else { + lines.extend([ + Assembly::inst2( + XCHG, + Ptr(RDI, size), + match size { + 1 => AL, + 2 => AX, + 4 => EAX, + _ => RAX, + }, + ), + Assembly::inst1(PUSH, 0), + ]); + } + } + lines.into() + } +} diff --git a/src/generator/x86_64/generator/globals.rs b/src/generator/x86_64/generator/globals.rs new file mode 100644 index 0000000..e84f444 --- /dev/null +++ b/src/generator/x86_64/generator/globals.rs @@ -0,0 +1,270 @@ +use super::AsmGenerator; +use crate::ctype::Type; +use crate::generator::Os; +use crate::generator::x86_64::assembly::Assembly; +use crate::global::{GlobalVariable, GlobalVariableData}; +use crate::token::Token; + +impl AsmGenerator<'_> { + pub(super) fn gen_string_literals(&self, string_literals: &[Token]) -> Assembly { + if string_literals.is_empty() { + vec![] + } else { + vec![ + if let Os::MacOS = self.target_os { + ".section __TEXT,__const".into() + } else { + ".section .data".into() + }, + string_literals + .iter() + .enumerate() + .map(|(i, token)| { + let bytes = token.encoded_string(); + vec![ + format!(" .balign {}", token.string_encoding.unit_size()).into(), + format!("L_.str.{}:", i).into(), + format!( + " .byte {}", + bytes + .into_iter() + .map(|byte| byte.to_string()) + .collect::>() + .join(", ") + ) + .into(), + ] + .into() + }) + .collect::>() + .into(), + ] + } + .into() + } + + pub(super) fn weak_reference(&self, name: &str) -> Assembly { + let directive = match self.target_os { + Os::MacOS => ".weak_reference", + Os::Linux => ".weak", + }; + format!("{directive} {}", self.with_prefix(name)).into() + } + + pub(super) fn gen_global_variable(&self, name: &str, gv: &GlobalVariable) -> Assembly { + if gv.declaration_only { + return if gv.weak_import { + self.weak_reference(name) + } else { + vec![].into() + }; + } + if gv.common && !gv.internal && !gv.thread_local && gv.data.is_none() { + let alignment = gv.ty.align_of().max(gv.alignment).max(1); + let alignment = match self.target_os { + Os::MacOS => alignment.trailing_zeros() as usize, + Os::Linux => alignment, + }; + return format!( + ".comm {},{},{}", + self.with_prefix(name), + gv.ty.size_of(), + alignment + ) + .into(); + } + vec![ + if let Os::MacOS = self.target_os { + ".section __DATA,__data".into() + } else { + ".section .data".into() + }, + if gv.internal { + vec![].into() + } else { + format!(".globl {}", self.with_prefix(name)).into() + }, + format!(" .balign {}", gv.ty.align_of().max(gv.alignment)).into(), + format!("{}:", self.with_prefix(name)).into(), + self.gen_initializer_element(&gv.ty, gv.data.as_ref()), + ] + .into() + } + + pub(super) fn gen_initializer_element( + &self, + ty: &Type, + data: Option<&GlobalVariableData>, + ) -> Assembly { + let data = if !matches!(ty, Type::Arr(..) | Type::Struct(_)) { + let mut value = data; + while let Some(GlobalVariableData::Arr(elements)) = value { + value = elements.first(); + } + value + } else { + data + }; + match ty { + Type::Aligned(child, _) => self.gen_initializer_element(child, data), + Type::Struct(struct_type) + if struct_type.has_bit_fields() || struct_type.definition().borrow().is_union => + { + let mut bytes = vec![0; ty.size_of()]; + let mut addresses = std::collections::HashMap::new(); + crate::global::initializer_bytes(ty, data, 0, &mut bytes, &mut addresses); + let mut lines = vec![]; + let mut index = 0; + while index < bytes.len() { + if let Some((symbol, delta)) = addresses.get(&index) { + let symbol = self.address_symbol(symbol); + lines.push(format!(" .8byte {symbol}{delta:+}").into()); + index += 8; + } else { + lines.push(format!(" .byte {}", bytes[index]).into()); + index += 1; + } + } + lines.into() + } + Type::Struct(struct_type) => { + let elements = if let Some(GlobalVariableData::Arr(elements)) = data { + Some(elements) + } else { + None + }; + let mut lines = vec![]; + let mut end = 0; + for (i, member) in struct_type + .members() + .iter() + .take(if struct_type.definition().borrow().is_union { + 1 + } else { + usize::MAX + }) + .enumerate() + { + if member.offset > end { + lines.push(format!(" .zero {}", member.offset - end).into()); + } + lines.push(self.gen_initializer_element( + &member.ty, + elements.and_then(|elements| elements.get(i)), + )); + end = member.offset + member.ty.size_of(); + } + if ty.size_of() > end { + lines.push(format!(" .zero {}", ty.size_of() - end).into()); + } + lines.into() + } + Type::Arr(children_ty, size) => { + let size = size.unwrap_or(0); + if let Some(GlobalVariableData::Arr(v)) = data { + vec![ + v.iter() + .enumerate() + .filter(|(i, _)| *i < size) + .map(|(_, d)| { + self.gen_initializer_element(children_ty.as_ref(), Some(d)) + }) + .collect::>() + .into(), + format!( + " .zero {}", + children_ty.size_of() + * v.iter() + .enumerate() + .filter(|(i, _)| *i < size) + .fold(size, |_, _| { size.saturating_sub(v.len()) }) + ) + .into(), + ] + .into() + } else { + format!(" .zero {}", children_ty.size_of() * size).into() + } + } + Type::Enum(value) => self.gen_initializer_element(&value.underlying.borrow(), data), + Type::I8 | Type::SignedChar => format!( + " .byte {}", + if let Some(GlobalVariableData::Elem(s)) = data { + s + } else { + "0" + } + ) + .into(), + Type::Integer128 { .. } => " .zero 16".into(), + Type::Float16 => " .zero 2".into(), + Type::Vector(..) => format!(" .zero {}", ty.size_of()).into(), + Type::LongDouble(bits) if *bits != 64 => { + let value = if let Some(GlobalVariableData::Elem(value)) = data { + value.parse::().unwrap() + } else { + 0 + }; + format!( + " .8byte {}\n .8byte {}", + value as u64, + (value >> 64) as u64 + ) + .into() + } + Type::Float(_) | Type::LongDouble(_) => format!( + " .{}byte {}", + ty.size_of(), + if let Some(GlobalVariableData::Elem(value)) = data { + value.as_str() + } else { + "0" + } + ) + .into(), + Type::Integer { .. } | Type::LongLong { .. } | Type::Bool => format!( + " {} {}", + if ty.size_of() == 1 { + ".byte".to_string() + } else { + format!(".{}byte", ty.size_of()) + }, + if let Some(GlobalVariableData::Address { symbol, offset }) = data { + format!("{}{offset:+}", self.address_symbol(symbol)) + } else if let Some(GlobalVariableData::Elem(value)) = data { + ty.convert_constant(value.parse().unwrap()).to_string() + } else { + "0".into() + } + ) + .into(), + Type::I32 => format!( + " .4byte {}", + if let Some(GlobalVariableData::Elem(s)) = data { + s + } else { + "0" + } + ) + .into(), + _ => format!( + " .8byte {}", + if let Some(GlobalVariableData::Address { symbol, offset }) = data { + let symbol = self.address_symbol(symbol); + format!("{symbol}{offset:+}") + } else if let Some(GlobalVariableData::Elem(s)) = data { + if s.starts_with("@.str.") { + s.replace('@', "L_") + } else if let Some(name) = s.strip_prefix('@') { + self.address_symbol(name) + } else { + s.clone() + } + } else { + "0".to_string() + } + ) + .into(), + } + } +} diff --git a/src/generator/x86_64/generator/inline_asm.rs b/src/generator/x86_64/generator/inline_asm.rs new file mode 100644 index 0000000..2bd9ef6 --- /dev/null +++ b/src/generator/x86_64/generator/inline_asm.rs @@ -0,0 +1,84 @@ +use super::AsmGenerator; +use crate::ctype::Type; +use crate::generator::x86_64::assembly::Assembly; +use crate::generator::x86_64::instruction::{InstOperand::*, InstOperator::*, Register::*}; +use crate::node::{Node, render_inline_assembly}; + +impl AsmGenerator<'_> { + pub(super) fn gen_inline_assembly( + &self, + node: &Node, + offset: usize, + breakable_branch_num: usize, + ) -> Assembly { + let registers = [RAX, R10, R11, RCX, RDX, RSI, RDI]; + let names = [ + ["%al", "%ax", "%eax", "%rax"], + ["%r10b", "%r10w", "%r10d", "%r10"], + ["%r11b", "%r11w", "%r11d", "%r11"], + ["%cl", "%cx", "%ecx", "%rcx"], + ["%dl", "%dx", "%edx", "%rdx"], + ["%sil", "%si", "%esi", "%rsi"], + ["%dil", "%di", "%edi", "%rdi"], + ]; + let operands = node.children.iter().chain(&node.args).collect::>(); + let template = render_inline_assembly(&node.dest, |index, modifier| { + let natural = operands.get(index)?.resolve_type()?.size_of(); + let width = crate::generator::asm_register_width( + crate::generator::Arch::X86_64, + modifier, + natural, + )?; + Some(names.get(index)?[width.trailing_zeros() as usize].into()) + }) + .unwrap_or_else(|| { + self.error_logger.print_error_position( + node.token.as_ref().unwrap().pos, + "unsupported asm register modifier for x86_64", + ); + unreachable!() + }); + let mut lines = vec![]; + let matching = node + .children + .iter() + .enumerate() + .filter(|(index, _)| node.value.unwrap() & (1 << index) != 0) + .collect::>(); + for (_, input) in &matching { + lines.push(self.gen_node(input, offset, breakable_branch_num)); + } + for input in &node.args { + lines.push(self.gen_node(input, offset, breakable_branch_num)); + } + for (index, input) in node + .args + .iter() + .enumerate() + .rev() + .map(|(index, input)| (index + node.children.len(), input)) + .chain(matching.into_iter().rev()) + { + let register = registers[index]; + lines.push(Assembly::inst1(POP, register)); + if input.resolve_type().unwrap().size_of() == 4 { + lines.push(Self::normalize_integer(&Type::integer(32, false), register)); + } + } + lines.push(Assembly::inst1(InlineAsm, InlineAsmTemplate(template))); + // Buffer every result before address generation can overwrite operand registers. + for register in registers[..node.children.len()].iter().rev() { + lines.push(Assembly::inst1(PUSH, *register)); + } + for output in &node.children { + lines.push(self.gen_addr(output, offset, breakable_branch_num)); + lines.extend([ + Assembly::inst1(POP, RAX), + Assembly::inst1(POP, RDI), + self.operation2rdi(output.resolve_type(), MOV, RAX), + ]); + } + lines.push(Assembly::inst1(PUSH, 0)); + lines.into() + } +}