mirror of
https://github.com/adambard/learnxinyminutes-docs.git
synced 2024-12-23 17:41:41 +00:00
Compare commits
7 Commits
eabefed759
...
6a18ee1f28
Author | SHA1 | Date | |
---|---|---|---|
|
6a18ee1f28 | ||
|
90d544271e | ||
|
fa0e3c632f | ||
|
9633245669 | ||
|
742574706b | ||
|
495272cff9 | ||
|
8c8790c7e1 |
218
dbt.html.markdown
Normal file
218
dbt.html.markdown
Normal file
@ -0,0 +1,218 @@
|
||||
---
|
||||
language: dbt
|
||||
filename: learndbt.sql
|
||||
contributors:
|
||||
- ["isabel faulds", "https://github.com/manyshapes"]
|
||||
|
||||
---
|
||||
|
||||
Data Build Tool or dbt™ (core) is an open-source command-line tool and framework
|
||||
for data transformation workflows. There is also a dbt cloud for managed, hosted
|
||||
dbt services with extended capabilities. dbt (core) utilizes .sql, .py, .yml,
|
||||
and .bash or any cli for orchestrating data warehouse command executions and
|
||||
generating .json artifacts.
|
||||
|
||||
dbt is agnostic to cloud provider, warehouse, and sql dialect. dbt works most
|
||||
effectively with version controlled systems and is implemented commonly with
|
||||
[git](git.html.markdown). dbt leverages [jinja](jinja.html.markdown) functions
|
||||
for dynamic values within [yaml](yaml.html.markdown), [sql](sql.html.markdown)
|
||||
and [python](python.html.markdown) .
|
||||
|
||||
### dbt SQL Models
|
||||
|
||||
```sql
|
||||
-- models can have optional configs specified within them
|
||||
{{ config(
|
||||
-- names in database will be filenames unless an alias
|
||||
alias='report' ,
|
||||
-- models can be views, tables, incremental (updated tables),
|
||||
-- ephemeral (temp tables), and snapshots (historical tables)
|
||||
materialized='incremental',
|
||||
-- if incremental, data capture methods can be specified with a strategy
|
||||
incremental_strategy='delete+insert',
|
||||
-- that can fail if the data schema changes
|
||||
on_schema_change='fail'
|
||||
--
|
||||
)}}
|
||||
|
||||
-- dbt emphasizes cte based development
|
||||
with staging_data as (
|
||||
select * ,
|
||||
current_timestamp() as loaded_date
|
||||
-- with refs to use other dbt seeds and models
|
||||
from {{ ref('seed_data') }}
|
||||
)
|
||||
|
||||
select
|
||||
_id,
|
||||
account_id,
|
||||
transaction_date,
|
||||
total_usd,
|
||||
loaded_date
|
||||
|
||||
from staging_data
|
||||
-- dbt can use {% if %} statements for conditional code block executions and
|
||||
-- is_incremental() to check if the current model is incremental
|
||||
{% if is_incremental() %}
|
||||
-- {{ this }} self references the current model
|
||||
where transaction_date > ( select max(transaction_date) from {{ this }} )
|
||||
-- the code block is not executed if the if statement is not met
|
||||
{% endif %}
|
||||
|
||||
|
||||
-- Snapshot models preserve historical data, or slowly changing dimensions
|
||||
{% snapshot snapshotted_model %}
|
||||
{{
|
||||
config(
|
||||
-- specifying a unique id
|
||||
unique_key='_id'
|
||||
-- dbt will create new records for snapshotted data if there are changes in the
|
||||
-- tracked columns
|
||||
strategy='check'
|
||||
check_cols=['account_manager'],
|
||||
snapshot_date='snapshot_date'
|
||||
)
|
||||
}}
|
||||
|
||||
with staging_data as (
|
||||
select * from {{ ref('staging_account_transactions')}}
|
||||
),
|
||||
|
||||
final as (
|
||||
select
|
||||
_id,
|
||||
transaction_date,
|
||||
account_id,
|
||||
-- when a change is detected in account_manager a new row will be saved
|
||||
account_manager,
|
||||
-- the row can have the more recent date of the snapshot for ordering
|
||||
CURRENT_TIMESTAMP() as snapshot_date,
|
||||
total_transaction_amount,
|
||||
-- saved, reusable sql operations can be performed with dbt macros
|
||||
{{ get_account_status('last_transaction_date', 'last_payment_date')
|
||||
}} as account_status,
|
||||
|
||||
from
|
||||
account_transactions
|
||||
)
|
||||
|
||||
select * from final
|
||||
{% endsnapshot %}
|
||||
|
||||
|
||||
-- macros are saved in files like macros/account_management_macros
|
||||
-- a macro is defined with the variables it's expecting
|
||||
{% macro get_account_status(last_transaction_date, last_payment_date) %}
|
||||
-- the sql saved for a macro will be performed on the given fields
|
||||
case
|
||||
when {{ last_transaction_date }} < {{last_payment_date}} and
|
||||
last_payment_date < CURRENT_DATE - INTERVAL '1 year' then 'Dormant'
|
||||
when {{ last_transaction_date }} > {{last_payment_date}} - INTERVAL
|
||||
'90 days' then 'Overdue'
|
||||
else 'Active'
|
||||
end
|
||||
-- % endmacro marks the end of the macro code block
|
||||
{% endmacro %}
|
||||
|
||||
```
|
||||
|
||||
## Configurations
|
||||
|
||||
```yml
|
||||
#########################################################
|
||||
# dbt_project.yml
|
||||
#########################################################
|
||||
#
|
||||
# cli commands are executed in same working directory as dbt_project.yml
|
||||
# dbt_project.yml will always have following values
|
||||
name: organization_project_name
|
||||
version: "1.0"
|
||||
profile: database_profile
|
||||
|
||||
# And many optional values with implicit defaults
|
||||
# like folder locations
|
||||
model-paths: ["models"]
|
||||
# or the output of the model
|
||||
models:
|
||||
organization_project:
|
||||
+materialized: view # Default materialization for models
|
||||
# these configurations are hierarchical and will act as defaults for files
|
||||
# without config blocks
|
||||
|
||||
#########################################################
|
||||
# profiles.yml
|
||||
#########################################################
|
||||
# The profile specified in dbt_project.yml is defined within `profiles.yml`
|
||||
database_profile:
|
||||
# Like all dbt files it can contain hard coded values
|
||||
target: hardcoded_target_environment_name
|
||||
outputs:
|
||||
dev:
|
||||
type: postgres
|
||||
# or environment variables using jinja
|
||||
user: "{{ env_var('POSTGRES_USER') }}"
|
||||
password: "{{ env_var('POSTGRES_PW') }}"
|
||||
# with defaults for variables if not available
|
||||
database: "{{ env_var('POSTGRES_DB', 'core') }}"
|
||||
# and python augmentation of variables
|
||||
schema: "{{ '_'.join([env_var('POSTGRES_USER').replace('.', '_').upper()
|
||||
, env_var('POSTGRES_SCHEMA') ]) }}"
|
||||
role: "{{ env_var('POSTGRES_ROLE')}}"
|
||||
```
|
||||
|
||||
### CLI Commands
|
||||
|
||||
```bash
|
||||
# cli commands are executed in same working directory as dbt_project.yml
|
||||
|
||||
# .csv files are seeded into database
|
||||
dbt seed
|
||||
# .sql or .py models are materialized in the database as tables or view
|
||||
dbt run
|
||||
# .sql or .yml tests can be performed
|
||||
dbt test
|
||||
# models can be materialized, ran, and shapshotted
|
||||
dbt build
|
||||
# a command can specify a model
|
||||
dbt build --select final_model
|
||||
# with upstream dependencies
|
||||
dbt build --select +final_model
|
||||
# and / or downstream dependencies
|
||||
dbt build --select +final_model+
|
||||
# metadata can be generated on materialized models
|
||||
dbt docs generate
|
||||
# full command list available in
|
||||
dbt list
|
||||
```
|
||||
|
||||
### Repository Structure
|
||||
|
||||
```text
|
||||
dbt has a default file structure when configurations do not define location
|
||||
|
||||
repository/
|
||||
└── dbt/
|
||||
├── dbt_project.yml # Required
|
||||
├── profiles.yml # Required
|
||||
├── models/ # Required , optional name
|
||||
│ ├── staging/ # Optional subfolders
|
||||
│ | └── staging_model.sql
|
||||
│ └── final_model.sql
|
||||
├── macros/ # Optional macro functions
|
||||
│ └── custom_macros.sql
|
||||
├── snapshots/ # Optional snapshot models
|
||||
│ └── snapshot.sql
|
||||
├── seeds/ # Optional csv files
|
||||
│ └── seed_data.csv
|
||||
├── logs/ # Output location
|
||||
├── target/ # Output location
|
||||
└── tests/ # Optional model tests
|
||||
└── custom_tests.sql
|
||||
```
|
||||
|
||||
## Further Reading
|
||||
|
||||
* [dbt logging](https://docs.getdbt.com/reference/global-configs/logs) - dbt documentation on outputs logs that can capture execution &
|
||||
debug logging
|
||||
* [dbt metadata artifacts](https://docs.getdbt.com/reference/artifacts/dbt-artifacts) - dbt documentation on generated artifacts, such as
|
||||
json documents for detailing attributes & metadata of a project
|
@ -30,7 +30,8 @@ program example ! declare a program called example.
|
||||
|
||||
! All declarations must come before statements and expressions.
|
||||
|
||||
implicit none ! prevents dynamic declaration of variables (recommended!)
|
||||
implicit none ! prevents dynamic declaration of variables
|
||||
! Recommended!
|
||||
! Implicit none must be redeclared in every function/program/module...
|
||||
|
||||
! IMPORTANT - Fortran is case insensitive.
|
||||
@ -45,10 +46,14 @@ program example ! declare a program called example.
|
||||
complex :: w = (0, 1) ! sqrt(-1)
|
||||
character(len=3) :: month ! string of 3 characters.
|
||||
|
||||
real :: array(6) ! declare an array of 6 reals.
|
||||
real, dimension(4) :: arrayb ! another way to declare an array.
|
||||
integer :: arrayc(-10:10) ! an array with a custom index.
|
||||
real :: array2d(3, 2) ! multidimensional array.
|
||||
! declare an array of 6 reals.
|
||||
real :: array(6)
|
||||
! another way to declare an array.
|
||||
real, dimension(4) :: arrayb
|
||||
! an array with a custom index -10 to 10 (inclusive)
|
||||
integer :: arrayc(-10:10)
|
||||
! A multidimensional array.
|
||||
real :: array2d(3, 2)
|
||||
|
||||
! The '::' separators are not always necessary but are recommended.
|
||||
|
||||
@ -77,7 +82,7 @@ program example ! declare a program called example.
|
||||
! Assignment & Arithmetic
|
||||
! =======================
|
||||
|
||||
Z = 1 ! assign to variable z declared above (case insensitive).
|
||||
Z = 1 ! assign to variable z declared above
|
||||
j = 10 + 2 - 3
|
||||
a = 11.54/(2.3*3.1)
|
||||
b = 2**3 ! exponentiation
|
||||
@ -86,7 +91,7 @@ program example ! declare a program called example.
|
||||
! ===================================
|
||||
|
||||
! Single-line if statement
|
||||
if (z == a) b = 4 ! condition always need surrounding parentheses.
|
||||
if (z == a) b = 4 ! conditions always need parentheses.
|
||||
|
||||
if (z /= a) then ! z not equal to a
|
||||
! Other symbolic comparisons are < > <= >= == /=
|
||||
@ -98,13 +103,13 @@ program example ! declare a program called example.
|
||||
b = 5 ! execution block must be on a new line.
|
||||
else
|
||||
b = 10
|
||||
end if ! end statement needs the 'if' (or can use 'endif').
|
||||
end if ! end statement needs the 'if'
|
||||
|
||||
if (.NOT. (x < c .AND. v >= a .OR. z == z)) then ! boolean operators.
|
||||
inner: if (.TRUE.) then ! can name if-construct.
|
||||
b = 1
|
||||
end if inner ! then must name endif statement.
|
||||
end if
|
||||
endif ! 'endif' is equivalent to 'end if'
|
||||
|
||||
i = 20
|
||||
select case (i)
|
||||
@ -128,16 +133,16 @@ program example ! declare a program called example.
|
||||
j = -1
|
||||
end select monthly
|
||||
|
||||
do i = 2, 10, 2 ! loops from 2 to 10 (inclusive) in increments of 2.
|
||||
do i = 2, 10, 2 ! loops from 2 to 10 (inclusive) in steps of 2.
|
||||
innerloop: do j = 1, 3 ! loops can be named too.
|
||||
exit ! quits the loop.
|
||||
end do innerloop
|
||||
cycle ! jump to next loop iteration.
|
||||
end do
|
||||
|
||||
! Goto statement exists but it is heavily discouraged though.
|
||||
! Goto statement exists but it is heavily discouraged.
|
||||
goto 10
|
||||
stop 1 ! stops code immediately (returning specified condition code).
|
||||
stop 1 ! stops the program, returns condition code 1.
|
||||
10 j = 201 ! this line is labeled as line 10
|
||||
|
||||
! Arrays
|
||||
@ -209,8 +214,12 @@ program example ! declare a program called example.
|
||||
|
||||
! we can have multiple format specifications.
|
||||
print "(I5,F6.2,E6.2)", 120, 43.41, 43.41
|
||||
print "(3I5)", 10, 20, 30 ! 3 repeats of integers (field width = 5).
|
||||
print "(2(I5,F6.2))", 120, 43.42, 340, 65.3 ! repeated grouping of formats.
|
||||
|
||||
! 3 repeats of integers (field width = 5).
|
||||
print "(3I5)", 10, 20, 30
|
||||
|
||||
! repeated grouping of formats.
|
||||
print "(2(I5,F6.2))", 120, 43.42, 340, 65.3
|
||||
|
||||
! We can also read input from the terminal.
|
||||
read (*, *) v
|
||||
@ -225,8 +234,9 @@ program example ! declare a program called example.
|
||||
|
||||
! To read a file.
|
||||
open (newunit=m, file="records.txt", status="old")
|
||||
! The file is referred to by a 'new unit number', an integer that the compiler
|
||||
! picks for you.
|
||||
! The file is referred to by a 'new unit number',
|
||||
! an integer that the compiler picks for you.
|
||||
|
||||
read (unit=m, fmt="(3F10.2)") a, b, c
|
||||
close (m)
|
||||
|
||||
@ -241,7 +251,7 @@ program example ! declare a program called example.
|
||||
call cpu_time(v) ! sets 'v' to a time in seconds.
|
||||
k = ior(i, j) ! bitwise OR of 2 integers.
|
||||
v = log10(x) ! log base 10.
|
||||
i = floor(b) ! returns the closest integer less than or equal to x.
|
||||
i = floor(b) ! converts b to integer by rounding down.
|
||||
v = aimag(w) ! imaginary part of a complex number.
|
||||
|
||||
! Functions & Subroutines
|
||||
@ -252,7 +262,7 @@ program example ! declare a program called example.
|
||||
|
||||
call routine(a, c, v) ! subroutine call.
|
||||
|
||||
! A function takes a list of input parameters and returns a single value.
|
||||
! A function takes several input parameters and returns a single value.
|
||||
! However the input parameters may still be modified and side effects
|
||||
! executed.
|
||||
|
||||
@ -261,21 +271,22 @@ program example ! declare a program called example.
|
||||
! Function calls can also be evoked within expressions.
|
||||
print *, func2(3, 2, k)
|
||||
|
||||
! A pure function is a function that doesn't modify its input parameters
|
||||
! or cause any side-effects.
|
||||
! A pure function is a function that doesn't modify its input
|
||||
! parameters or cause any side-effects.
|
||||
m = func3(3, 2, k)
|
||||
|
||||
contains ! Zone for defining sub-programs internal to the program.
|
||||
contains ! Start defining the program's internal procedures:
|
||||
|
||||
! Fortran has a couple of slightly different ways to define functions.
|
||||
|
||||
integer function func(a, b, c) ! a function returning an integer value.
|
||||
! implicit none ! subvariable fields can no longer declare implicit none
|
||||
integer, intent(in) :: a, b, c ! type of input parameters defined inside the function.
|
||||
! implicit none ! - no longer used in subvariable fields
|
||||
integer, intent(in) :: a, b, c ! type of input parameters
|
||||
! the return variable defaults to the function name.
|
||||
|
||||
if (a >= 2) then
|
||||
func = a + b + c ! the return variable defaults to the function name.
|
||||
return ! can return the current value from the function at any time.
|
||||
func = a + b + c
|
||||
return ! returns the current value at 'func'
|
||||
end if
|
||||
func = a + c
|
||||
|
||||
@ -286,17 +297,20 @@ contains ! Zone for defining sub-programs internal to the pro
|
||||
integer, intent(in) :: a, b ! can declare and enforce that variables
|
||||
!are not modified by the function.
|
||||
integer, intent(inout) :: c
|
||||
integer :: f ! function return type declared inside the function.
|
||||
integer :: cnt = 0 ! GOTCHA - initialisation implies variable is
|
||||
integer :: f
|
||||
! function return type declared inside the function.
|
||||
integer :: cnt = 0 ! GOTCHA -
|
||||
! assigning a value at initalization
|
||||
! implies that the variable is
|
||||
! saved between function calls.
|
||||
|
||||
f = a + b - c
|
||||
c = 4 ! altering the value of an input variable.
|
||||
c = 4 ! changing value of input variable c.
|
||||
cnt = cnt + 1 ! count number of function calls.
|
||||
|
||||
end function func2
|
||||
|
||||
pure function func3(a, b, c) ! a pure function can have no side-effects.
|
||||
pure function func3(a, b, c) ! a pure function has no side-effects.
|
||||
integer, intent(in) :: a, b, c
|
||||
integer :: func3
|
||||
|
||||
@ -304,6 +318,8 @@ contains ! Zone for defining sub-programs internal to the pro
|
||||
|
||||
end function func3
|
||||
|
||||
! a subroutine does not return anything,
|
||||
! but can change the value of arguments.
|
||||
subroutine routine(d, e, f)
|
||||
real, intent(inout) :: f
|
||||
real, intent(in) :: d, e
|
||||
@ -312,7 +328,8 @@ contains ! Zone for defining sub-programs internal to the pro
|
||||
|
||||
end subroutine routine
|
||||
|
||||
end program example ! End of Program Definition -----------------------
|
||||
end program example
|
||||
! End of Program Definition -----------------------
|
||||
|
||||
! Functions and Subroutines declared externally to the program listing need
|
||||
! to be declared to the program using an Interface declaration (even if they
|
||||
@ -350,7 +367,8 @@ module fruity
|
||||
use fruit, only: apple, pear ! use apple and pear from fruit module.
|
||||
implicit none ! comes after module imports.
|
||||
|
||||
private ! make things private to the module (default is public).
|
||||
! By default all module data and functions will be public
|
||||
private ! Instead set default to private
|
||||
! Declare some variables/functions explicitly public.
|
||||
public :: apple, mycar, create_mycar
|
||||
! Declare some variables/functions private to the module (redundant here).
|
||||
|
@ -44,6 +44,9 @@ Multi-line comments look like this.
|
||||
import java.util.ArrayList;
|
||||
// Import all classes inside of java.security package
|
||||
import java.security.*;
|
||||
// Java to illustrate calling of static members and methods without calling classname
|
||||
import static java.lang.Math.*;
|
||||
import static java.lang.System.*;
|
||||
|
||||
public class LearnJava {
|
||||
|
||||
@ -211,9 +214,21 @@ public class LearnJava {
|
||||
// Prefer the String constructor when you need an exact value.
|
||||
BigDecimal tenCents = new BigDecimal("0.1");
|
||||
|
||||
// Type inference with 'var'
|
||||
var x = 100; // int
|
||||
var y = 1.90; // double
|
||||
var z = 'a'; // char
|
||||
var p = "tanu"; // String
|
||||
var q = false; // boolean
|
||||
|
||||
// Strings
|
||||
String fooString = "My String Is Here!";
|
||||
|
||||
// Text blocks
|
||||
vat textBlock = """
|
||||
This is a <Text Block> in Java
|
||||
""";
|
||||
|
||||
// \n is an escaped character that starts a new line
|
||||
String barString = "Printing on a new line?\nNo Problem!";
|
||||
// \t is an escaped character that adds a tab character
|
||||
@ -459,6 +474,8 @@ public class LearnJava {
|
||||
System.out.println(br.readLine());
|
||||
// In Java 7, the resource will always be closed, even if it throws
|
||||
// an Exception.
|
||||
} catch (IOException | SQLException ex) {
|
||||
// Java 7+ Multi catch block handle both exceptions
|
||||
} catch (Exception ex) {
|
||||
//The resource will be closed before the catch statement executes.
|
||||
System.out.println("readLine() failed.");
|
||||
@ -852,6 +869,12 @@ public abstract class Mammal()
|
||||
}
|
||||
}
|
||||
|
||||
// Java Records are a concise way to define immutable data carrier classes, automatically
|
||||
// generating boilerplate code like constructors, equals(), hashCode()and toString().
|
||||
// This automatically creates an immutable class Person with fields name and age.
|
||||
public record Person(String name, int age) {}
|
||||
Person p = new Person("Alice", 30);
|
||||
|
||||
// Enum Type
|
||||
//
|
||||
// An enum type is a special data type that enables for a variable to be a set
|
||||
|
@ -11,6 +11,7 @@ Jinja is a fast, expressive, and extensible templating engine for Python
|
||||
applications.
|
||||
|
||||
Jinja includes a lot of functionalities, such as:
|
||||
|
||||
- Template inheritance and inclusion;
|
||||
- Defining and importing macros within templates;
|
||||
- Security mechanisms to prevent XSS attacks;
|
||||
|
@ -342,7 +342,7 @@ void main() {
|
||||
We define a new input variable ```color``` which represents our color data, this data
|
||||
is passed on to ```fColor```, which is an output variable of our vertex shader and
|
||||
becomes an input variable for our fragment shader.
|
||||
It is imporatant that variables passed between shaders have the exact same name
|
||||
It is important that variables passed between shaders have the exact same name
|
||||
and type.
|
||||
|
||||
## Handling VBO's
|
||||
@ -574,7 +574,7 @@ in vec2 fTexCoords;
|
||||
out vec4 outColor;
|
||||
|
||||
void main() {
|
||||
// texture() loads the current texure data at the specified texture coords,
|
||||
// texture() loads the current texture data at the specified texture coords,
|
||||
// then we can simply multiply them by our color.
|
||||
outColor = texture(tex, fTexCoords) * vec4(fColor, 1.0);
|
||||
}
|
||||
@ -685,7 +685,7 @@ Geometry shaders are inbetween the vertex and the fragment shader.
|
||||
|
||||
layout(location = 0) in vec3 position;
|
||||
layout(location = 1) in vec3 color;
|
||||
// Create an output interface block passed to the next shadaer stage.
|
||||
// Create an output interface block passed to the next shader stage.
|
||||
// Interface blocks can be used to structure data passed between shaders.
|
||||
out VS_OUT {
|
||||
vec3 color;
|
||||
|
Loading…
Reference in New Issue
Block a user